def test_split_tuple_iterator():
X, y = utils.split_tuple_generator(
lambda: [("val1", "label1"), ("val2", "label2"), ("val3", "label3")])
assert list(X()) == ["val1", "val2", "val3"]
assert list(y) == ["label1", "label2", "label3"]
assert list(y) == ["label1", "label2", "label3"]
assert list(X()) == ["val1", "val2", "val3"]
assert list(y) == ["label1", "label2", "label3"]
def train(self, importance_cutoff=0.15, limit=None):
classes, self.class_names = self.get_labels()
self.class_names = sort_class_names(self.class_names)
# Get items and labels, filtering out those for which we have no labels.
X_gen, y = split_tuple_generator(lambda: self.items_gen(classes))
# Extract features from the items.
X = self.extraction_pipeline.fit_transform(X_gen)
# Calculate labels.
y = np.array(y)
if limit:
X = X[:limit]
y = y[:limit]
print(f"X: {X.shape}, y: {y.shape}")
is_multilabel = isinstance(y[0], np.ndarray)
is_binary = len(self.class_names) == 2
# Split dataset in training and test.
X_train, X_test, y_train, y_test = self.train_test_split(X, y)
if self.sampler is not None:
pipeline = make_pipeline(self.sampler, self.clf)
else:
pipeline = self.clf
tracking_metrics = {}
# Use k-fold cross validation to evaluate results.
if self.cross_validation_enabled:
scorings = ["accuracy"]
if len(self.class_names) == 2:
scorings += ["precision", "recall"]
scores = cross_validate(pipeline,
X_train,
y_train,
scoring=scorings,
cv=5)
print("Cross Validation scores:")
for scoring in scorings:
score = scores[f"test_{scoring}"]
tracking_metrics[f"test_{scoring}"] = {
"mean": score.mean(),
"std": score.std() * 2,
}
print(
f"{scoring.capitalize()}: f{score.mean()} (+/- {score.std() * 2})"
)
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
# Training on the resampled dataset if sampler is provided.
if self.sampler is not None:
X_train, y_train = self.sampler.fit_resample(X_train, y_train)
print(
f"resampled X_train: {X_train.shape}, y_train: {y_train.shape}"
)
print(f"X_test: {X_test.shape}, y_test: {y_test.shape}")
self.clf.fit(X_train, y_train)
print("Model trained")
feature_names = self.get_human_readable_feature_names()
if self.calculate_importance and len(feature_names):
explainer = shap.TreeExplainer(self.clf)
shap_values = explainer.shap_values(X_train)
# In the binary case, sometimes shap returns a single shap values matrix.
if is_binary and not isinstance(shap_values, list):
shap_values = [-shap_values, shap_values]
summary_plot_value = shap_values[1]
summary_plot_type = "layered_violin"
else:
summary_plot_value = shap_values
summary_plot_type = None
shap.summary_plot(
summary_plot_value,
to_array(X_train),
feature_names=feature_names,
class_names=self.class_names,
plot_type=summary_plot_type,
show=False,
)
matplotlib.pyplot.savefig("feature_importance.png",
bbox_inches="tight")
matplotlib.pyplot.xlabel("Impact on model output")
matplotlib.pyplot.clf()
important_features = self.get_important_features(
importance_cutoff, shap_values)
self.print_feature_importances(important_features)
# Save the important features in the metric report too
feature_report = self.save_feature_importances(
important_features, feature_names)
tracking_metrics["feature_report"] = feature_report
print("Training Set scores:")
y_pred = self.clf.predict(X_train)
if not is_multilabel:
print(
classification_report_imbalanced(y_train,
y_pred,
labels=self.class_names))
print("Test Set scores:")
# Evaluate results on the test set.
y_pred = self.clf.predict(X_test)
if is_multilabel:
assert isinstance(
y_pred[0], np.ndarray), "The predictions should be multilabel"
print(f"No confidence threshold - {len(y_test)} classified")
if is_multilabel:
confusion_matrix = metrics.multilabel_confusion_matrix(
y_test, y_pred)
else:
confusion_matrix = metrics.confusion_matrix(
y_test, y_pred, labels=self.class_names)
print(
classification_report_imbalanced(y_test,
y_pred,
labels=self.class_names))
report = classification_report_imbalanced_values(
y_test, y_pred, labels=self.class_names)
tracking_metrics["report"] = report
print_labeled_confusion_matrix(confusion_matrix,
self.class_names,
is_multilabel=is_multilabel)
tracking_metrics["confusion_matrix"] = confusion_matrix.tolist()
confidence_thresholds = [0.6, 0.7, 0.8, 0.9]
if is_binary:
confidence_thresholds = [0.1, 0.2, 0.3, 0.4
] + confidence_thresholds
# Evaluate results on the test set for some confidence thresholds.
for confidence_threshold in confidence_thresholds:
y_pred_probas = self.clf.predict_proba(X_test)
confidence_class_names = self.class_names + ["__NOT_CLASSIFIED__"]
y_pred_filter = []
classified_indices = []
for i in range(0, len(y_test)):
if not is_binary:
argmax = np.argmax(y_pred_probas[i])
else:
argmax = 1 if y_pred_probas[i][
1] > confidence_threshold else 0
if y_pred_probas[i][argmax] < confidence_threshold:
if not is_multilabel:
y_pred_filter.append("__NOT_CLASSIFIED__")
continue
classified_indices.append(i)
if is_multilabel:
y_pred_filter.append(y_pred[i])
else:
y_pred_filter.append(argmax)
if not is_multilabel:
y_pred_filter = np.array(y_pred_filter)
y_pred_filter[classified_indices] = self.le.inverse_transform(
np.array(y_pred_filter[classified_indices], dtype=int))
classified_num = sum(1 for v in y_pred_filter
if v != "__NOT_CLASSIFIED__")
print(
f"\nConfidence threshold > {confidence_threshold} - {classified_num} classified"
)
if is_multilabel:
confusion_matrix = metrics.multilabel_confusion_matrix(
y_test[classified_indices], np.asarray(y_pred_filter))
else:
confusion_matrix = metrics.confusion_matrix(
y_test.astype(str),
y_pred_filter.astype(str),
labels=confidence_class_names,
)
print(
classification_report_imbalanced(
y_test.astype(str),
y_pred_filter.astype(str),
labels=confidence_class_names,
))
print_labeled_confusion_matrix(confusion_matrix,
confidence_class_names,
is_multilabel=is_multilabel)
self.evaluation()
if self.entire_dataset_training:
print("Retraining on the entire dataset...")
if self.sampler is not None:
X_train, y_train = self.sampler.fit_resample(X, y)
else:
X_train = X
y_train = y
print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
self.clf.fit(X_train, y_train)
with open(self.__class__.__name__.lower(), "wb") as f:
pickle.dump(self, f, protocol=pickle.HIGHEST_PROTOCOL)
if self.store_dataset:
with open(f"{self.__class__.__name__.lower()}_data_X", "wb") as f:
pickle.dump(X, f, protocol=pickle.HIGHEST_PROTOCOL)
with open(f"{self.__class__.__name__.lower()}_data_y", "wb") as f:
pickle.dump(y, f, protocol=pickle.HIGHEST_PROTOCOL)
return tracking_metrics
