def residuals(regressor, X, y):
# Create the train and test splits
X_train, X_test, y_train, y_test = model_selection.train_test_split(
X, y, test_size=0.2)
# Store labels and colors for the legend ordered by call
regressor.fit(X_train, y_train)
train_score_ = regressor.score(X_train, y_train)
test_score_ = regressor.score(X_test, y_test)
y_pred_train = regressor.predict(X_train)
residuals_train = y_pred_train - y_train
y_pred_test = regressor.predict(X_test)
residuals_test = y_pred_test - y_test
table = make_table(
y_pred_train,
residuals_train,
y_pred_test,
residuals_test,
train_score_,
test_score_,
)
chart = wandb.visualize("wandb/residuals_plot/v1", table)
return chart
def learning_curve(
model,
X,
y,
cv=None,
shuffle=False,
random_state=None,
train_sizes=None,
n_jobs=1,
scoring=None,
):
"""Trains model on datasets of varying size and generates plot of score vs size.
Called by plot_learning_curve to visualize learning curve. Please use the function
plot_learning_curve() if you wish to visualize your learning curves.
"""
train_sizes, train_scores, test_scores = model_selection.learning_curve(
model,
X,
y,
cv=cv,
n_jobs=n_jobs,
train_sizes=train_sizes,
scoring=scoring,
shuffle=shuffle,
random_state=random_state,
)
train_scores_mean = np.mean(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
table = make_table(train_scores_mean, test_scores_mean, train_sizes)
chart = wandb.visualize("wandb/learning_curve/v1", table)
return chart
def roc_table(fpr_dict, tpr_dict, classes, indices_to_plot):
data = []
count = 0
for i, to_plot in enumerate(indices_to_plot):
fpr_dict[i], tpr_dict[i], _ = roc_curve(y_true,
probas[:, i],
pos_label=classes[i])
if to_plot:
roc_auc = auc(fpr_dict[i], tpr_dict[i])
for j in range(len(fpr_dict[i])):
if labels is not None and (
isinstance(classes[i], int)
or isinstance(classes[0], np.integer)):
class_dict = labels[classes[i]]
else:
class_dict = classes[i]
fpr = [
class_dict,
round(fpr_dict[i][j], 3),
round(tpr_dict[i][j], 3)
]
data.append(fpr)
count += 1
if count >= chart_limit:
wandb.termwarn(
"wandb uses only the first %d datapoints to create the plots."
% wandb.Table.MAX_ROWS)
break
return wandb.visualize(
'wandb/roc/v1',
wandb.Table(columns=['class', 'fpr', 'tpr'], data=data))
def heatmap_table(x_labels, y_labels, matrix_values, show_text):
x_axis = []
y_axis = []
values = []
count = 0
for i, x in enumerate(x_labels):
for j, y in enumerate(y_labels):
x_axis.append(x)
y_axis.append(y)
values.append(matrix_values[j][i])
count += 1
if count >= chart_limit:
wandb.termwarn(
"wandb uses only the first %d datapoints to create the plots."
% wandb.Table.MAX_ROWS)
break
heatmap_key = "wandb/heatmap/v1" if show_text else "wandb/heatmap_no_text/v1"
return wandb.visualize(
heatmap_key,
wandb.Table(
columns=["x_axis", "y_axis", "values"],
data=[[x_axis[i], y_axis[i],
round(values[i], 2)] for i in range(len(x_axis))],
),
)
def decision_boundaries(decision_boundary_x, decision_boundary_y,
decision_boundary_color, train_x, train_y,
train_color, test_x, test_y, test_color):
x_dict = []
y_dict = []
color_dict = []
shape_dict = []
for i in range(min(len(decision_boundary_x),100)):
x_dict.append(decision_boundary_x[i])
y_dict.append(decision_boundary_y[i])
color_dict.append(decision_boundary_color)
for i in range(300):
x_dict.append(test_x[i])
y_dict.append(test_y[i])
color_dict.append(test_color[i])
for i in range(min(len(train_x),600)):
x_dict.append(train_x[i])
y_dict.append(train_y[i])
color_dict.append(train_color[i])
return wandb.visualize(
'wandb/decision_boundaries/v1', wandb.Table(
columns=['x', 'y', 'color'],
data=[
[x_dict[i], y_dict[i], color_dict[i]] for i in range(len(x_dict))
]
))
def residuals(y_pred_train, residuals_train, y_pred_test, residuals_test, train_score_, test_score_):
y_pred_dict = []
dataset_dict = []
residuals_dict = []
datapoints = 0
max_datapoints_train = 900
max_datapoints_train = 100
for pred, residual in zip(y_pred_train, residuals_train):
# add class counts from training set
y_pred_dict.append(pred)
dataset_dict.append("train")
residuals_dict.append(residual)
datapoints += 1
if(datapoints >= max_datapoints_train):
wandb.termwarn("wandb uses only the first %d datapoints to create the plots."% wandb.Table.MAX_ROWS)
break
datapoints = 0
for pred, residual in zip(y_pred_test, residuals_test):
# add class counts from training set
y_pred_dict.append(pred)
dataset_dict.append("test")
residuals_dict.append(residual)
datapoints += 1
if(datapoints >= max_datapoints_train):
wandb.termwarn("wandb uses only the first %d datapoints to create the plots."% wandb.Table.MAX_ROWS)
break
return wandb.visualize(
'wandb/residuals_plot/v1', wandb.Table(
columns=['dataset', 'y_pred', 'residuals', 'train_score', 'test_score'],
data=[
[dataset_dict[i], y_pred_dict[i], residuals_dict[i], train_score_, test_score_] for i in range(len(y_pred_dict))
]
))
def decision_boundaries(
decision_boundary_x,
decision_boundary_y,
decision_boundary_color,
train_x,
train_y,
train_color,
test_x,
test_y,
test_color,
):
x_dict, y_dict, color_dict = [], [], []
for i in range(min(len(decision_boundary_x), 100)):
x_dict.append(decision_boundary_x[i])
y_dict.append(decision_boundary_y[i])
color_dict.append(decision_boundary_color)
for i in range(300):
x_dict.append(test_x[i])
y_dict.append(test_y[i])
color_dict.append(test_color[i])
for i in range(min(len(train_x), 600)):
x_dict.append(train_x[i])
y_dict.append(train_y[i])
color_dict.append(train_color[i])
return wandb.visualize(
"wandb/decision_boundaries/v1",
wandb.Table(
columns=["x", "y", "color"],
data=[[x_dict[i], y_dict[i], color_dict[i]]
for i in range(len(x_dict))],
),
)
def class_proportions(classes_, class_counts_train, class_counts_test):
class_dict = []
dataset_dict = []
count_dict = []
for i in range(len(classes_)):
# add class counts from training set
class_dict.append(classes_[i])
dataset_dict.append("train")
count_dict.append(class_counts_train[i])
# add class counts from test set
class_dict.append(classes_[i])
dataset_dict.append("test")
count_dict.append(class_counts_test[i])
if i >= chart_limit:
wandb.termwarn("wandb uses only the first %d datapoints to create the plots."% wandb.Table.MAX_ROWS)
break
if labels is not None and (isinstance(class_dict[0], int)
or isinstance(class_dict[0], np.integer)):
class_dict = get_named_labels(labels, class_dict)
return wandb.visualize(
'wandb/class_proportions/v1', wandb.Table(
columns=['class', 'dataset', 'count'],
data=[
[class_dict[i], dataset_dict[i], count_dict[i]] for i in range(len(class_dict))
]
))
def pr_table(pr_curves):
data = []
count = 0
for i, class_name in enumerate(pr_curves.keys()):
precision, recall = pr_curves[class_name]
for p, r in zip(precision, recall):
# if class_names are ints and labels are set
if labels is not None and (isinstance(class_name, int)
or isinstance(
class_name, np.integer)):
class_name = labels[class_name]
# if class_names are ints and labels are not set
# or, if class_names have something other than ints
# (string, float, date) - user class_names
data.append([class_name, round(p, 3), round(r, 3)])
count += 1
if count >= chart_limit:
wandb.termwarn(
"wandb uses only the first %d datapoints to create the plots."
% wandb.Table.MAX_ROWS)
break
return wandb.visualize(
'wandb/pr_curve/v1',
wandb.Table(columns=['class', 'precision', 'recall'],
data=data))
def feature_importances_table(feature_names, importances):
return wandb.visualize(
'wandb/feature_importances/v1', wandb.Table(
columns=['feature_names', 'importances'],
data=[
[feature_names[i], importances[i]] for i in range(len(feature_names))
]
))
def calibration_curves(model_dict, frac_positives_dict, mean_pred_value_dict, hist_dict, edge_dict):
return wandb.visualize(
'wandb/calibration/v1', wandb.Table(
columns=['model', 'fraction_of_positives', 'mean_predicted_value', 'hist_dict', 'edge_dict'],
data=[
[model_dict[i], frac_positives_dict[i], mean_pred_value_dict[i], hist_dict[i], edge_dict[i]] for i in range(len(model_dict))
]
))
def outlier_candidates(distance, outlier_percentage, influence_threshold):
return wandb.visualize(
'wandb/outliers/v1', wandb.Table(
columns=['distance', 'instance_indicies', 'outlier_percentage', 'influence_threshold'],
data=[
[distance[i], i, round_3(outlier_percentage_), influence_threshold_] for i in range(len(distance))
]
))
def summary_metrics(model=None, X=None, y=None, X_test=None, y_test=None):
"""
Calculates summary metrics (like mse, mae, r2 score) for both regression and
classification algorithms.
Called by plot_summary_metrics to visualize metrics. Please use the function
plot_summary_metric() if you wish to visualize your summary metrics.
"""
if (test_missing(model=model, X=X, y=y, X_test=X_test, y_test=y_test) and
test_types(model=model, X=X, y=y, X_test=X_test, y_test=y_test) and
test_fitted(model)):
y = np.asarray(y)
y_test = np.asarray(y_test)
metric_name=[]
metric_value=[]
model_name = model.__class__.__name__
params = {}
# Log model params to wandb.config
for v in vars(model):
if isinstance(getattr(model, v), str) \
or isinstance(getattr(model, v), bool) \
or isinstance(getattr(model, v), int) \
or isinstance(getattr(model, v), float):
params[v] = getattr(model, v)
# Classifier Metrics
if sklearn.base.is_classifier(model):
y_pred = model.predict(X_test)
y_probas = model.predict_proba(X_test)
metric_name.append("accuracy_score")
metric_value.append(round_2(sklearn.metrics.accuracy_score(y_test, y_pred)))
metric_name.append("precision")
metric_value.append(round_2(sklearn.metrics.precision_score(y_test, y_pred, average="weighted")))
metric_name.append("recall")
metric_value.append(round_2(sklearn.metrics.recall_score(y_test, y_pred, average="weighted")))
metric_name.append("f1_score")
metric_value.append(round_2(sklearn.metrics.f1_score(y_test, y_pred, average="weighted")))
# Regression Metrics
elif sklearn.base.is_regressor(model):
y_pred = model.predict(X_test)
metric_name.append("mae")
metric_value.append(round_2(sklearn.metrics.mean_absolute_error(y_test, y_pred)))
metric_name.append("mse")
metric_value.append(round_2(sklearn.metrics.mean_squared_error(y_test, y_pred)))
metric_name.append("r2_score")
metric_value.append(round_2(sklearn.metrics.r2_score(y_test, y_pred)))
return wandb.visualize(
'wandb/metrics/v1', wandb.Table(
columns=['metric_name', 'metric_value', 'model_name'],
data= [
[metric_name[i], metric_value[i], model_name] for i in range(len(metric_name))
]
))
def elbow_curve(cluster_ranges, clfs, times):
return wandb.visualize(
'wandb/elbow/v1',
wandb.Table(
columns=['cluster_ranges', 'errors', 'clustering_time'],
data=[
[cluster_ranges[i], clfs[i], times[i]] for i in range(len(cluster_ranges))
]
))
def silhouette(x, y, colors, centerx, centery, y_sil, x_sil, color_sil, silhouette_avg):
return wandb.visualize(
'wandb/silhouette_/v1', wandb.Table(
columns=['x', 'y', 'colors', 'centerx', 'centery', 'y_sil', 'x1', 'x2', 'color_sil', 'silhouette_avg'],
data=[
[x[i], y[i], colors[i], None, None,
y_sil[i], 0, x_sil[i], color_sil[i], silhouette_avg]
for i in range(len(color_sil))
]
))
def confusion_matrix(
y_true=None,
y_pred=None,
labels=None,
true_labels=None,
pred_labels=None,
normalize=False,
):
"""Computes the confusion matrix to evaluate the performance of a classification.
Called by plot_confusion_matrix to visualize roc curves. Please use the function
plot_confusion_matrix() if you wish to visualize your confusion matrix.
"""
cm = metrics.confusion_matrix(y_true, y_pred)
if labels is None:
classes = unique_labels(y_true, y_pred)
else:
classes = np.asarray(labels)
if normalize:
cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis]
cm = np.around(cm, decimals=2)
cm[np.isnan(cm)] = 0.0
if true_labels is None:
true_classes = classes
else:
validate_labels(classes, true_labels, "true_labels")
true_label_indexes = np.in1d(classes, true_labels)
true_classes = classes[true_label_indexes]
cm = cm[true_label_indexes]
if pred_labels is None:
pred_classes = classes
else:
validate_labels(classes, pred_labels, "pred_labels")
pred_label_indexes = np.in1d(classes, pred_labels)
pred_classes = classes[pred_label_indexes]
cm = cm[:, pred_label_indexes]
table = make_table(cm, pred_classes, true_classes, labels)
chart = wandb.visualize("wandb/confusion_matrix/v1", table)
return chart
def learning_curve_table(train, test, trainsize):
data=[]
for i in range(len(train)):
if i >= chart_limit/2:
wandb.termwarn("wandb uses only the first %d datapoints to create the plots."% wandb.Table.MAX_ROWS)
break
train_set = ["train", round(train[i],2), trainsize[i]]
test_set = ["test", round(test[i],2), trainsize[i]]
data.append(train_set)
data.append(test_set)
return wandb.visualize(
'wandb/learning_curve/v1', wandb.Table(
columns=['dataset', 'score', 'train_size'],
data=data
))
def elbow_curve(clusterer, X, cluster_ranges, n_jobs, show_cluster_time):
if cluster_ranges is None:
cluster_ranges = range(1, 10, 2)
else:
cluster_ranges = sorted(cluster_ranges)
clfs, times = _compute_results_parallel(n_jobs, clusterer, X,
cluster_ranges)
clfs = np.absolute(clfs)
table = make_table(cluster_ranges, clfs, times)
chart = wandb.visualize("wandb/elbow/v1", table)
return chart
def confusion_matrix_table(cm, label):
data = []
count = 0
pred_classes, true_classes = [label, 'Rest'], [label, 'Rest']
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
pred_dict = pred_classes[i]
true_dict = true_classes[j]
data.append([pred_dict, true_dict, cm[i, j]])
count += 1
if count >= wandb.Table.MAX_ROWS:
wandb.termwarn(
f"wandb uses only the first {wandb.Table.MAX_ROWS} datapoints to create plots."
)
break
return wandb.visualize(
'wandb/confusion_matrix/v1',
wandb.Table(columns=['Predicted', 'Actual', 'Count'], data=data))
def feature_importances(model, feature_names):
attributes_to_check = [
"feature_importances_", "feature_log_prob_", "coef_"
]
found_attribute = check_for_attribute_on(model, attributes_to_check)
if found_attribute is None:
wandb.termwarn(
f"could not find any of attributes {', '.join(attributes_to_check)} on classifier. Cannot plot feature importances."
)
return
elif found_attribute == "feature_importances_":
importances = model.feature_importances_
elif found_attribute == "coef_": # ElasticNet-like models
importances = model.coef_
elif found_attribute == "feature_log_prob_":
# coef_ was deprecated in sklearn 0.24, replaced with
# feature_log_prob_
importances = model.feature_log_prob_
if len(importances.shape) > 1:
n_significant_dims = sum([i > 1 for i in importances.shape])
if n_significant_dims > 1:
nd = len(importances.shape)
wandb.termwarn(
f"{nd}-dimensional feature importances array passed to plot_feature_importances. "
f"{nd}-dimensional and higher feature importances arrays are not currently supported. "
f"These importances will not be plotted.")
return
else:
importances = np.squeeze(importances)
indices = np.argsort(importances)[::-1]
importances = importances[indices]
if feature_names is None:
feature_names = indices
else:
feature_names = np.array(feature_names)[indices]
table = make_table(feature_names, importances)
chart = wandb.visualize("wandb/feature_importances/v1", table)
return chart
def summary_metrics(model=None, X=None, y=None, X_test=None, y_test=None):
"""Calculates summary metrics for both regressors and classifiers.
Called by plot_summary_metrics to visualize metrics. Please use the function
plot_summary_metrics() if you wish to visualize your summary metrics.
"""
y, y_test = np.asarray(y), np.asarray(y_test)
metrics = {}
model_name = model.__class__.__name__
y_pred = model.predict(X_test)
if sklearn.base.is_classifier(model):
accuracy_score = sklearn.metrics.accuracy_score(y_test, y_pred)
metrics["accuracy_score"] = accuracy_score
precision = sklearn.metrics.precision_score(y_test, y_pred, average="weighted")
metrics["precision"] = precision
recall = sklearn.metrics.recall_score(y_test, y_pred, average="weighted")
metrics["recall"] = recall
f1_score = sklearn.metrics.f1_score(y_test, y_pred, average="weighted")
metrics["f1_score"] = f1_score
elif sklearn.base.is_regressor(model):
mae = sklearn.metrics.mean_absolute_error(y_test, y_pred)
metrics["mae"] = mae
mse = sklearn.metrics.mean_squared_error(y_test, y_pred)
metrics["mse"] = mse
r2_score = sklearn.metrics.r2_score(y_test, y_pred)
metrics["r2_score"] = r2_score
metrics = {name: utils.round_2(metric) for name, metric in metrics.items()}
table = make_table(metrics, model_name)
chart = wandb.visualize("wandb/metrics/v1", table)
return chart
def outlier_candidates(regressor, X, y):
# Fit a linear model to X and y to compute MSE
regressor.fit(X, y)
# Leverage is computed as the diagonal of the projection matrix of X
leverage = (X * np.linalg.pinv(X).T).sum(1)
# Compute the rank and the degrees of freedom of the OLS model
rank = np.linalg.matrix_rank(X)
df = X.shape[0] - rank
# Compute the MSE from the residuals
residuals = y - regressor.predict(X)
mse = np.dot(residuals, residuals) / df
# Compute Cook's distance
residuals_studentized = residuals / np.sqrt(mse) / np.sqrt(1 - leverage)
distance_ = residuals_studentized**2 / X.shape[1]
distance_ *= leverage / (1 - leverage)
# Compute the influence threshold rule of thumb
influence_threshold_ = 4 / X.shape[0]
outlier_percentage_ = sum(distance_ >= influence_threshold_) / X.shape[0]
outlier_percentage_ *= 100.0
distance_dict, count = [], 0
for d in distance_:
distance_dict.append(d)
count += 1
if utils.check_against_limit(
count,
"outlier_candidates",
utils.chart_limit,
):
break
table = make_table(distance_dict, outlier_percentage_,
influence_threshold_)
chart = wandb.visualize("wandb/outliers/v1", table)
return chart
def confusion_matrix_table(cm, pred_classes, true_classes):
data=[]
count = 0
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
if labels is not None and (isinstance(pred_classes[i], int)
or isinstance(pred_classes[0], np.integer)):
pred_dict = labels[pred_classes[i]]
true_dict = labels[true_classes[j]]
else:
pred_dict = pred_classes[i]
true_dict = true_classes[j]
data.append([pred_dict, true_dict, cm[i,j]])
count+=1
if count >= chart_limit:
wandb.termwarn("wandb uses only the first %d datapoints to create the plots."% wandb.Table.MAX_ROWS)
break
return wandb.visualize(
'wandb/confusion_matrix/v1', wandb.Table(
columns=['Predicted', 'Actual', 'Count'],
data=data
))
def class_proportions(y_train, y_test, labels):
# Get the unique values from the dataset
targets = (y_train,) if y_test is None else (y_train, y_test)
class_ids = np.array(unique_labels(*targets))
# Compute the class counts
counts_train = np.array([(y_train == c).sum() for c in class_ids])
counts_test = np.array([(y_test == c).sum() for c in class_ids])
class_column, dataset_column, count_column = make_columns(
class_ids, counts_train, counts_test
)
if labels is not None and (
isinstance(class_column[0], int) or isinstance(class_column[0], np.integer)
):
class_column = get_named_labels(labels, class_column)
table = make_table(class_column, dataset_column, count_column)
chart = wandb.visualize("wandb/class_proportions/v1", table)
return chart
def silhouette(clusterer, X, cluster_labels, labels, metric, kmeans):
# Run clusterer for n_clusters in range(len(cluster_ranges), get cluster labels
# TODO - keep/delete once we decide if we should train clusterers
# or ask for trained models
# clusterer.set_params(n_clusters=n_clusters, random_state=42)
# cluster_labels = clusterer.fit_predict(X)
cluster_labels = np.asarray(cluster_labels)
labels = np.asarray(labels)
le = LabelEncoder()
_ = le.fit_transform(cluster_labels)
n_clusters = len(np.unique(cluster_labels))
# The silhouette_score gives the average value for all the samples.
# This gives a perspective into the density and separation of the formed
# clusters
silhouette_avg = silhouette_score(X, cluster_labels, metric=metric)
# Compute the silhouette scores for each sample
sample_silhouette_values = silhouette_samples(X,
cluster_labels,
metric=metric)
x_sil, y_sil, color_sil = [], [], []
count, y_lower = 0, 10
for i in range(n_clusters):
# Aggregate the silhouette scores for samples belonging to
# cluster i, and sort them
ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels
== i]
ith_cluster_silhouette_values.sort()
size_cluster_i = ith_cluster_silhouette_values.shape[0]
y_upper = y_lower + size_cluster_i
y_values = np.arange(y_lower, y_upper)
for j in range(len(y_values)):
y_sil.append(y_values[j])
x_sil.append(ith_cluster_silhouette_values[j])
color_sil.append(i)
count += 1
if utils.check_against_limit(count, "silhouette",
utils.chart_limit):
break
# Compute the new y_lower for next plot
y_lower = y_upper + 10 # 10 for the 0 samples
if kmeans:
centers = clusterer.cluster_centers_
centerx = centers[:, 0]
centery = centers[:, 1]
else:
centerx = [None] * len(color_sil)
centery = [None] * len(color_sil)
table = make_table(
X[:, 0],
X[:, 1],
cluster_labels,
centerx,
centery,
y_sil,
x_sil,
color_sil,
silhouette_avg,
)
chart = wandb.visualize("wandb/silhouette_/v1", table)
return chart
