def print_prec_rec(y_test, pred):
'''
y_test are the labels for the test data
pred is the prediction
prints a precision recall diagram for a binary classification prediction
'''
precision, recall, _ = precision_recall_curve(y_test, pred)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(pyplot.fill_between).parameters else {})
pyplot.step(recall, precision, color='b', alpha=0.2, where='post')
pyplot.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
print(type(recall))
print(recall)
print(type(precision))
print(precision)
pyplot.xlabel('Recall')
pyplot.ylabel('Precision')
pyplot.ylim([0.0, 1.05])
pyplot.xlim([0.0, 1.0])
pyplot.show()
def _handle_extra_syntax_parameters(self, params):
"""
Handles extra parameters given to the constructor such as
*columns* or a role.
"""
# remove column_ for roles
def clean_name(name):
return DataRoles._allowed_attr.get(name, name)
set_params = set(map(clean_name, params))
# Checks that extra parameters are allowed.
sign = signature(self.__class__.__init__)
allowed = set(sign.parameters)
notin = set_params - allowed - \
BasePipelineItem._hidden_constructor_arguments
if len(notin) > 0:
allowed = "\n".join(
wrap(", ".join(sorted(filter(lambda _: _ != 'self',
allowed)))))
if len(notin) == 1:
raise NameError("Parameter '{0}' is not allowed for class '{"
"1}'.\nAllowed: {2}".format(
list(sorted(notin))[0],
self.__class__.__name__, allowed))
else:
raise NameError("Parameters {0} are not allowed for class '{"
"1}'.\nAllowed: {2}".format(
sorted(notin), self.__class__.__name__,
allowed))
# Handles parameters columns.
inputs = OrderedDict()
cols = params.pop('columns', None)
if cols:
if isinstance(cols, dict):
inputs.update(cols)
else:
self.set_inputs(cols, early=True)
for role in DataRoles._allowed:
name = DataRoles.to_attribute(role)
if name in params:
if cols is not None and role in cols and params[name] != \
cols[role]:
raise AttributeError(
"Attribute '{0}' is already set to '{1}', "
"cannot be replaced by '{2}'".format(
name, cols[role], params[name]))
attr = DataRoles.to_attribute(role)
if attr in allowed:
setattr(self, attr, params[name])
else:
inputs[role] = params[name]
del params[name]
if len(inputs) > 0:
self.set_inputs(inputs, early=True)
def plot_precision_recall_curve(input_shp,groud_truth_shp,save_path):
# from sklearn.metrics import precision_recall_curve
from sklearn.utils.fixes import signature
precision, recall, _ = precision_recall_curve_iou(input_shp,groud_truth_shp)
average_precision = calculate_average_precision(precision,recall)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_pos = 'mid' # post
step_kwargs = ({'step': step_pos}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where=step_pos)
plt.plot(recall, precision, 'r--')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([-0.01, 1.05])
plt.xlim([-0.01, 1.01])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
# save average_precision to txt file
txt_path = os.path.splitext(save_path)[0]+'_ap.txt'
with open(txt_path,'w') as f_obj:
f_obj.writelines('shape_file average_precision\n')
f_obj.writelines('%s %.4lf\n' % (input_shp, average_precision))
# plt.show()
plt.savefig(save_path,dpi=300)
basic.outputlogMessage("Output figures to %s" % os.path.abspath(save_path))
def plot_precision_recall_curve(clf, X_test, y_test):
if hasattr(clf, 'decision_function'):
y_scores = clf.decision_function(X_test)
elif hasattr(clf, 'predict_proba'):
y_scores = clf.predict_proba(X_test)[:, 1]
else:
print('Precision recall curve not possible')
return
average_precision = average_precision_score(y_test, y_scores)
print('Average precision-recall score: {0:0.2f}'.format(average_precision))
precision, recall, thresholds = precision_recall_curve(y_test, y_scores)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
def precision_recall_plot(targs, preds, figsize=(6,6)):
"""
Plots the precision recall curve
----------
targs: array-like true class labels
preds: array-like predicted probabilities
figsize: size of figure
Returns:
-------
A precision and recall curve
"""
average_precision = average_precision_score(targs, preds)
precision, recall, _ = precision_recall_curve(targs, preds)
plt.figure(figsize=figsize)
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
plt.show()
def plot_precision_recall(classifier, X_test, y_test, y_score):
""" Function to plot an precision-recall curve knowing the features, resp and model """
from sklearn.metrics import precision_recall_curve
import matplotlib.pyplot as plt
from sklearn.utils.fixes import signature
from sklearn.metrics import average_precision_score
classifier.fit(X_train, y_train)
predictions_probs = classifier.predict_proba(X_test)
average_precision = average_precision_score(y_test, predictions_probs[:,1])
print('Average precision-recall score: {0:0.2f}'.format(
average_precision))
precision, recall, _ = precision_recall_curve(y_test, predictions_probs[:,1])
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
def plotAUPRCForLogisticRegression(X_train, y_train, testData, y_pred,
predictionProbability):
import matplotlib.pyplot as plt4
# Create a simple classifier
classifier = svm.LinearSVC()
classifier.fit(X_train, y_train)
y_score = classifier.decision_function(testData)
from sklearn.metrics import average_precision_score
average_precision = average_precision_score(y_train, y_score)
from sklearn.metrics import precision_recall_curve
import matplotlib.pyplot as plt
from sklearn.utils.fixes import signature
precision, recall, _ = precision_recall_curve(y_train,
predictionProbability)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt4.step(recall, precision, color='b', alpha=0.2, where='post')
plt4.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt4.xlabel('Recall')
plt4.ylabel('Precision')
plt4.ylim([0.0, 1.05])
plt4.xlim([0.0, 1.0])
plt4.title(
'2-class Precision-Recall(Logistic Regression) curve: AP= {}'.format(
average_precision))
plt4.show()
def _get_param_names(cls):
"""Get parameter names for the estimator"""
# fetch the constructor or the original constructor before
# deprecation wrapping if any
init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
if init is object.__init__:
# No explicit constructor to introspect
return []
# introspect the constructor arguments to find the model parameters
# to represent
init_signature = signature(init)
# Consider the constructor parameters excluding 'self'
parameters = [
p for p in init_signature.parameters.values()
if p.name != 'self' and p.kind != p.VAR_KEYWORD
]
# Extract and sort argument names excluding 'self'
parameters = set([p.name for p in parameters])
# recurse
for superclass in cls.__bases__:
try:
parameters.update(superclass._get_param_names())
except AttributeError:
# object and pygsp.graphs.Graph don't have this method
pass
return parameters
def draw_pr_curve(y_true, y_pred, **kwargs):
precision, recall, _ = precision_recall_curve(y_true, y_pred)
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
title = kwargs.get('title', None)
plot_path = kwargs.get('plot_path', None)
curve_auc = auc(recall, precision)
if title is None:
plt.title('AUC=%.2f' % curve_auc)
else:
plt.title('%s, AUC=%.2f' % (title, curve_auc))
if plot_path is None:
plt.show()
else:
plt.savefig(plot_path)
plt.close()
return curve_auc
def _check_roles(self):
"""
Checks the consistency between defined roles and supported roles.
"""
if not hasattr(self, '_entrypoint'):
raise SystemExit(
'One internal learner does not follow the new syntax.')
params = signature(self._entrypoint).parameters
for role in DataRoles._allowed:
attr = DataRoles.to_attribute(role)
if hasattr(self, attr) and getattr(self, attr) is not None and \
attr not in params:
if role == Role.Label:
# warnings instead of an exception but we should
# really simplify the logic
# in experiment.py. The model should know which
# roles it supports.
# current code makes it difficult to guess.
# A minor modification in entrypoints.py should do the
# trick.
if self.type != "clusterer":
warnings.warn(
"Model '{0}' (type='{1}') does not support "
"role '{2}' (for developers, check "
"_allowed_roles is defined).".format(
type(self), self.type, role))
else:
raise RuntimeError(
"Model '{0}' (type='{1}') does not support role "
"'{2}' (for developers, check _allowed_roles is "
"defined).".format(type(self), self.type, role))
def precisionRecall_microJoint_curve(self, precision, recall):
'''
Function to plot precision recall curve with micro averaged scores
# args
precision: list of precision score for each class along with micro averaged score
recall: list of recall score for each class along with micro averaged score
# returns
precision recall curve
'''
plt.figure()
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall['micro'],
precision['micro'],
color='b',
alpha=0.2,
where='post')
plt.fill_between(recall["micro"],
precision["micro"],
alpha=0.2,
color='b',
**step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Micro-averaged over all classes')
return plt
def drawPRC(result, sp, methodlabel):
''' Accept a result dataframe whose 'cond' coulmn is binary series
representing the ture condition and 'pred' column is the normalized
score of predicton.
'''
print("drawing prc curve...")
sp.set_xlabel('Recall')
sp.set_ylabel('Precision')
sp.set_ylim([0.0, 1.0])
sp.set_xlim([0.0, 1.0])
sp.set_title('2-class Precision-Recall curve')
precision, recall, threshold = precision_recall_curve(
result['cond'], result['pred'])
average_precision = average_precision_score(result['cond'], result['pred'])
myauc = auc(recall, precision)
step_kwargs = ({
'step': 'post'
} if 'step' in signature(sp.fill_between).parameters else {})
sp.step(recall, precision, alpha=0.2, where='post')
sp.fill_between(recall,
precision,
alpha=0.2,
label=methodlabel + " (AUC = %.3f)" % myauc,
**step_kwargs)
return myauc
def draw_precision_recall_curve(classifier, x_data, y_data):
''' Takes: classifier object, feature and target data,
Returns: chart showing precision recall curve
Reference: code drawn from Scikit-learn documentation, https://bit.ly/2WaYP2I
'''
if isinstance(classifier, LinearSVC):
pred_scores = classifier.decision_function(x_data)
else:
pred_scores = classifier.predict_proba(x_data)[:, 1]
precision, recall, thresholds = precision_recall_curve(y_data, pred_scores)
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Precision-Recall Curve')
return
def _build_repr(self):
# XXX This is copied from sklearn.BaseEstimator's get_params
cls = self.__class__
init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
init_signature = signature(init)
if init is object.__init__:
args = []
else:
args = sorted([p.name for p in init_signature.parameters.values()
if p.name != 'self' and p.kind != p.VAR_KEYWORD])
class_name = self.__class__.__name__
params = dict()
for key in args:
warnings.simplefilter("always", DeprecationWarning)
try:
with warnings.catch_warnings(record=True) as w:
value = getattr(self, key, None)
if len(w) and w[0].category == DeprecationWarning:
continue
finally:
warnings.filters.pop(0)
params[key] = value
return '%s(%s)' % (class_name, _pprint(params, offset=len(class_name)))
def _get_param_names(cls):
"""Get parameter names for the estimator"""
# fetch the constructor or the original constructor before
# deprecation wrapping if any
init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
if init is object.__init__:
# No explicit constructor to introspect
return []
# introspect the constructor arguments to find the model parameters
# to represent
from sklearn.utils.fixes import signature
init_signature = signature(init)
# Consider the constructor parameters excluding 'self'
parameters = [p for p in init_signature.parameters.values()
if p.name != 'self' and p.kind != p.VAR_KEYWORD]
for p in parameters:
if p.kind == p.VAR_POSITIONAL:
raise RuntimeError("scikit-learn estimators should always "
"specify their parameters in the signature"
" of their __init__ (no varargs)."
" %s with constructor %s doesn't "
" follow this convention."
% (cls, init_signature))
# Extract and sort argument names excluding 'self'
return sorted([p.name for p in parameters])
def test(model):
_, _, x_score, y_score = get_data()
y_pred = []
for im in x_score:
im = im.reshape(2304)
pred = np.dot(im, model.W) + model.b
y_pred.append(pred)
precision, recall, _ = precision_recall_curve(y_score, y_pred)
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
fig2 = plt.figure()
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
y_score = (y_score > 0.5)
y_pred = (np.asarray(y_pred) > 0.5)
ACA = accuracy_score(y_score, y_pred, normalize=True)
f1 = f1_score(y_score, y_pred, 'binary')
print('ACA; ' + str(ACA))
print('F1; ' + str(f1))
plt.title('2-class Precision-Recall curve: ACA={0:0.2f}'.format(ACA) +
' F1={0:0.2f}'.format(f1))
plt.show()
def plot_positionalPRC(positionalPRC_output):
'''
accepts output dictionary from the positionalPRC function of the form: motif_name --> [precision,recall,auPRC]
generates PRC curves for each motif on same coordinates
'''
from sklearn.utils.fixes import signature
for motif_name, values in positionalPRC_output.items():
recall = values[0]
precision = values[1]
auPRC = str(round(values[2], 3))
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall,
precision,
label=motif_name + ":" + auPRC,
where='post')
#uncomment to fill the area below the curve, generally not desirable if multiple curves plotted on same axes.
#plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.legend()
plt.show()
def evaluation(self, df_true, df_predict, df_prob, output_eval_dir):
run = Run.get_context()
# precition-recall-curve
average_precision = average_precision_score(df_true, df_predict)
precision, recall, _ = precision_recall_curve(df_true, df_prob)
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
f1_plt = plt.figure(1)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall,
precision,
alpha=0.2,
color='b',
**step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0, 1.1])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
run.log_image("precition/recall curve", plot=f1_plt)
f1_plt.savefig(os.path.join(output_eval_dir, 'precition_recall.png'))
f2_plt = plt.figure(2)
metrics_name = ['precition', 'recall', 'F1-Score']
p = precision_score(df_true, df_predict, average='binary')
r = recall_score(df_true, df_predict, average='binary')
f1 = f1_score(df_true, df_predict, average='binary')
values_list = [p, r, f1]
plt.bar(metrics_name,
values_list,
width=0.8,
facecolor="#ff9999",
edgecolor="white")
for x, y in zip(metrics_name, values_list):
plt.text(x, y, '%.4f' % y, ha='center', va='bottom')
plt.ylim([0, 1.1])
plt.ylabel('score')
plt.title('Scores')
run.log_image("scores", plot=f2_plt)
f2_plt.savefig(os.path.join(output_eval_dir, 'scores.png'))
f3_plt = plt.figure(3)
# Compute fpr, tpr, thresholds and roc auc
fpr, tpr, thresholds = roc_curve(df_true, df_prob)
roc_auc = auc(df_true, df_prob)
plt.plot(fpr, tpr, label='ROC curve (area = %0.3f)' % roc_auc)
plt.plot([0, 1], [0, 1], 'k--') # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate or (1 - Specifity)')
plt.ylabel('True Positive Rate or (Sensitivity)')
plt.title('ROC Curve')
plt.legend(loc="lower right")
run.log_image("ROC curve", plot=f3_plt)
f3_plt.savefig(os.path.join(output_eval_dir, 'roc.png'))
def plot_precision_recall_curve(labels,
predictions_prob,
class_nums,
average_function='weighted'):
"""
Plot precision recall curve
:param labels:
:param predictions_prob:
:param class_nums:
:param average_function:
:return:
"""
labels = label_binarize(labels,
classes=np.linspace(0,
class_nums - 1,
num=class_nums).tolist())
predictions_prob = np.array(predictions_prob, dtype=np.float32)
precision = dict()
recall = dict()
average_precision = dict()
for i in range(class_nums):
precision[i], recall[i], _ = precision_recall_curve(
labels[:, i], predictions_prob[:, i])
average_precision[i] = average_precision_score(labels[:, i],
predictions_prob[:, i])
precision[average_function], recall[
average_function], _ = precision_recall_curve(labels.ravel(),
predictions_prob.ravel())
average_precision[average_function] = average_precision_score(
labels, predictions_prob, average=average_function)
log.info('Average precision score, {:s}-averaged '
'over all classes: {:.5f}'.format(
average_function, average_precision[average_function]))
plt.figure()
plt.step(recall[average_function],
precision[average_function],
color='b',
alpha=0.2,
where='post')
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.fill_between(recall[average_function],
precision[average_function],
alpha=0.2,
color='b',
**step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Average precision score, {:s}-averaged over '
'all classes: AP={:.5f}'.format(
average_function, average_precision[average_function]))
def plot_pr_curve(y, yhat):
precision, recall, _ = precision_recall_curve(y.ravel(), yhat.ravel())
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.grid(color='b', linestyle='--', linewidth=0.5, alpha=0.3)
def test_kernel_theta():
# Check that parameter vector theta of kernel is set correctly.
for kernel in kernels:
if isinstance(kernel, KernelOperator) \
or isinstance(kernel, Exponentiation): # skip non-basic kernels
continue
theta = kernel.theta
_, K_gradient = kernel(X, eval_gradient=True)
# Determine kernel parameters that contribute to theta
init_sign = signature(kernel.__class__.__init__).parameters.values()
args = [p.name for p in init_sign if p.name != 'self']
theta_vars = map(lambda s: s[0:-len("_bounds")],
filter(lambda s: s.endswith("_bounds"), args))
assert_equal(
set(hyperparameter.name
for hyperparameter in kernel.hyperparameters),
set(theta_vars))
# Check that values returned in theta are consistent with
# hyperparameter values (being their logarithms)
for i, hyperparameter in enumerate(kernel.hyperparameters):
assert_equal(theta[i],
np.log(getattr(kernel, hyperparameter.name)))
# Fixed kernel parameters must be excluded from theta and gradient.
for i, hyperparameter in enumerate(kernel.hyperparameters):
# create copy with certain hyperparameter fixed
params = kernel.get_params()
params[hyperparameter.name + "_bounds"] = "fixed"
kernel_class = kernel.__class__
new_kernel = kernel_class(**params)
# Check that theta and K_gradient are identical with the fixed
# dimension left out
_, K_gradient_new = new_kernel(X, eval_gradient=True)
assert_equal(theta.shape[0], new_kernel.theta.shape[0] + 1)
assert_equal(K_gradient.shape[2], K_gradient_new.shape[2] + 1)
if i > 0:
assert_equal(theta[:i], new_kernel.theta[:i])
assert_array_equal(K_gradient[..., :i],
K_gradient_new[..., :i])
if i + 1 < len(kernel.hyperparameters):
assert_equal(theta[i + 1:], new_kernel.theta[i:])
assert_array_equal(K_gradient[..., i + 1:],
K_gradient_new[..., i:])
# Check that values of theta are modified correctly
for i, hyperparameter in enumerate(kernel.hyperparameters):
theta[i] = np.log(42)
kernel.theta = theta
assert_almost_equal(getattr(kernel, hyperparameter.name), 42)
setattr(kernel, hyperparameter.name, 43)
assert_almost_equal(kernel.theta[i], np.log(43))
def get_params(self, deep=True):
"Scikit-learn API, returns all parameters."
sig = signature(self.__class__.__init__)
params = [(p if p != 'columns' else '_columns', p)
for p in sig.parameters if p not in ('self', 'params')]
res = {p: getattr(self, att)
for att, p in params if hasattr(self, att)}
if hasattr(self, "_columns") and isinstance(self._columns, dict):
res['columns'] = self._columns
return res
def prec_rec_curve(prec, rec, avg_prec):
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(rec, prec, color='b', alpha=0.2, where='post')
plt.fill_between(rec, prec, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(avg_prec))
def plot_pr_curve(self, y_pred, y_score):
precision, recall, thresholds = precision_recall_curve(self.y_test, y_score)
step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, here='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
average_precision = average_precision_score(self.y_test, y_score)
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))
plt.show()
return plt
def plot_prec_recall(precision, recall, thresholds):
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve')
def site_example():
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.show()
def has_fit_parameter(estimator, parameter):
"""Checks whether the estimator's fit method supports the given parameter.
Examples
--------
>>> from sklearn.svm import SVC
>>> has_fit_parameter(SVC(), "sample_weight")
True
"""
# print("has_fit_parameter has been called")
addfreq()
return parameter in signature(estimator.fit).parameters
def validate(self):
if len(self.models) == 0:
raise Exception('Model has not been initialized')
index = 1
AP = []
Majority = []
val_X = self.feature.val_X.copy()
models = self.models
for model in models:
y_true = self.feature.val_y.copy()
y_true[y_true != index] = 0
y_true[y_true != 0] = 1
if hasattr(model, 'decision_function'):
y_pred = model.decision_function(val_X)
else:
y_pred = model.predict(val_X)
average_precision = average_precision_score(y_true, y_pred)
AP.append(average_precision)
index += 1
from sklearn.metrics import precision_recall_curve
import matplotlib.pyplot as plt
from sklearn.utils.fixes import signature
precision, recall, threshold = precision_recall_curve(
y_true, y_pred)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({
'step': 'post'
} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall,
precision,
alpha=0.2,
color='b',
**step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
plt.savefig('AUC' + self.feature.feat_types[0] + str(index) +
'.png')
mAP = np.mean(np.array(AP))
return AP, mAP
def clf_print_results(clf, Xtrain, ytrain, Xtest, ytest):
"""
Classifier results:
* params:
- clf: classifier
- Xtrain, ytrain, Xtest, ytest
* prints:
- classification report
- accuracy score
- confustion matrix
- AUC for ROC and ROC curve
- AUC for precision recall and precision-recall curve
"""
clf.fit(Xtrain, ytrain)
ypred = clf.predict(Xtest)
print("Classification report:\n", classification_report(ytest,ypred))
accuracy = accuracy_score(ytest, ypred)
conf_mat = confusion_matrix(ytest, ypred)
print("Accuracy %.2f%%" % (accuracy*100))
print("---- Confusion matrix ----")
print(conf_mat)
ypred_proba = clf.predict_proba(Xtest)[:,1]
fpr, tpr, thr_roc = roc_curve(ytest, ypred_proba)
auc_roc = auc(fpr,tpr)
precision, recall, thr_pr = precision_recall_curve(ytest, ypred_proba)
auc_pr = auc(recall, precision)
average_precision = average_precision_score(ytest, ypred_proba)
#----------------- FIGURES AUC -----------------
fig, ax = plt.subplots(1,2, figsize=(14,6))
ax[0].plot(fpr, tpr, '-', lw=2, label='AUC-ROC=%.3f' % auc_roc)
ax[0].plot([0, 1], [0, 1],linestyle='--')
ax[0].set_xlabel('False Positive Rate', fontsize=16)
ax[0].set_ylabel('True Positive Rate', fontsize=16)
ax[0].set_title('ROC Curve',fontsize=16)
ax[0].legend(loc="lower right", fontsize=14)
plt.subplots_adjust(hspace = 0.3, top=0.92)
step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post', label='AUC-PR=%.3f' % auc_pr)
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
ax[1].set_xlabel('Recall', fontsize=16)
ax[1].set_ylabel('Precision', fontsize=16)
ax[1].set_title('Precision-Recall Curve', fontsize=16)
#ax[1].plot([0, 1], [0.1, 0.1], linestyle='--')
ax[1].legend(loc="upper right", fontsize=14)
plt.show()
def basic_assess_AUC(scores, labels, plot_pr_idx=None):
assert len(scores) == len(labels)
if plot_pr_idx is not None:
precision, recall, _ = precision_recall_curve(labels, scores[:, plot_pr_idx])
print(len(np.where(labels == 0)[0]), len(np.where(labels == 1)[0]), len(np.unique(precision)), len(np.unique(recall)))
step_kwargs = ({'step': 'post'} if 'step' in signature(plt.fill_between).parameters else {})
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('recall')
plt.ylabel('precision')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.show()
return np.array([roc_auc_score(labels, scores[:, i]) for i in range(scores.shape[1])]),
np.array([average_precision_score(labels, scores[:, i]) for i in range(scores.shape[1])])
def plot_prc(plot_file, y_test, y_score, average_precision, txt):
precision, recall, _ = sklearn.metrics.precision_recall_curve(y_test, y_score)
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('{0} AP={1:0.2f}'.format(txt,
average_precision))
plt.axes().set_aspect('equal')
plt.savefig(plot_file, bbox_inches='tight')
def _get_args(function, varargs=False):
"""Helper to get function arguments"""
try:
params = signature(function).parameters
except ValueError:
# Error on builtin C function
return []
args = [key for key, param in params.items()
if param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)]
if varargs:
varargs = [param.name for param in params.values()
if param.kind == param.VAR_POSITIONAL]
if len(varargs) == 0:
varargs = None
return args, varargs
else:
return args
def _serialize_cross_validator(o):
ret = OrderedDict()
parameters = OrderedDict()
# XXX this is copied from sklearn.model_selection._split
cls = o.__class__
init = getattr(cls.__init__, 'deprecated_original', cls.__init__)
# Ignore varargs, kw and default values and pop self
init_signature = signature(init)
# Consider the constructor parameters excluding 'self'
if init is object.__init__:
args = []
else:
args = sorted([p.name for p in init_signature.parameters.values()
if p.name != 'self' and p.kind != p.VAR_KEYWORD])
for key in args:
# We need deprecation warnings to always be on in order to
# catch deprecated param values.
# This is set in utils/__init__.py but it gets overwritten
# when running under python3 somehow.
warnings.simplefilter("always", DeprecationWarning)
try:
with warnings.catch_warnings(record=True) as w:
value = getattr(o, key, None)
if len(w) and w[0].category == DeprecationWarning:
# if the parameter is deprecated, don't show it
continue
finally:
warnings.filters.pop(0)
if not (hasattr(value, '__len__') and len(value) == 0):
value = json.dumps(value)
parameters[key] = value
else:
parameters[key] = None
ret['oml-python:serialized_object'] = 'cv_object'
name = o.__module__ + "." + o.__class__.__name__
value = OrderedDict([['name', name], ['parameters', parameters]])
ret['value'] = value
return ret
def plot_positionalPRC(positionalPRC_output):
'''
accepts output dictionary from the positionalPRC function of the form: motif_name --> [precision,recall,auPRC]
generates PRC curves for each motif on same coordinates
'''
from sklearn.utils.fixes import signature
for motif_name,values in positionalPRC_output.items():
recall=values[0]
precision=values[1]
auPRC=str(round(values[2],3))
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, label=motif_name+":"+auPRC,where='post')
#uncomment to fill the area below the curve, generally not desirable if multiple curves plotted on same axes.
#plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.legend()
plt.show()
print('Average precision-recall score: {0:0.2f}'.format(
average_precision))
###############################################################################
# Plot the Precision-Recall curve
# ................................
from sklearn.metrics import precision_recall_curve
import matplotlib.pyplot as plt
from sklearn.utils.fixes import signature
precision, recall, _ = precision_recall_curve(y_test, y_score)
# In matplotlib < 1.5, plt.fill_between does not have a 'step' argument
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
###############################################################################
# In multi-label settings
# ------------------------
