def log_image_dir_snapshots(image_dir, channel_name='image_dir_snapshots', experiment=None, sample=16, seed=1234): """Logs visual snapshot of the directory with image data to Neptune. For a given directory with images it logs a sample of images as figure to Neptune. If the `image_dir` specified contains multiple folders it will sample per folder and create multiple figures naming each figure with the folder name. See snapshots per class here https://ui.neptune.ai/jakub-czakon/examples/e/EX-95/channels. Args: image_dir(str): path to directory with images. sample(int): number of images that should be sampled for plotting. channel_name(str): name of the neptune channel. Default is 'image_dir_snapshots'. experiment(neptune.experiemnts.Experiment or None): if the data should be logged to a particular neptune experiment it can be passed here. By default it is logged to the current experiment. seed(int): random state for the sampling of images. Examples: Initialize Neptune:: import neptune from neptunecontrib.versioning.data import log_image_dir_snapshots neptune.init('USER_NAME/PROJECT_NAME') Log visual snapshot of image directory:: PATH = 'train_dir/' with neptune.create_experiment(): log_image_dir_snapshots(PATH) """ _exp = experiment if experiment else neptune figs = _get_collated_images(image_dir, sample=sample, seed=seed) for fig in figs: send_figure(fig, channel_name=channel_name, experiment=_exp)
def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''): """Creates and logs KS statistics curve and KS statistics score to Neptune. Kolmogorov-Smirnov statistics chart can be calculated for true positive rates (TPR) and true negative rates (TNR) for each threshold and plotted on a chart. The maximum distance from TPR to TNR can be treated as performance metric. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Create and log KS statistics curve and KS statistics score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_ks_statistic neptune.init() with neptune.create_experiment(): log_ks_statistic(y_test, y_test_pred) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune res = binary_ks_curve(y_true, y_pred[:, 1]) ks_stat = res[3] _exp.log_metric(prefix + 'ks_statistic', ks_stat) fig, ax = plt.subplots() plt_metrics.plot_ks_statistic(y_true, y_pred, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def draw_concept_symbol_matrix(self): figure, ax = plt.subplots(figsize=(20, 5)) figure.suptitle(f'Concept-symbol matrix {self.epoch_counter}') g = sns.heatmap(self.concept_symbol_matrix, annot=True, fmt='.2f', ax=ax) g.set_title(f'Concept-symbol matrix {self.epoch_counter}') send_figure(figure, channel_name=self.prefix + 'concept_symbol_matrix') plt.close()
def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None, channel_name='metrics_by_threshold', prefix=''): """Creates metric/threshold charts for each metric and logs them to Neptune. Metrics for which charsta re created and logged are: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value', 'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate' Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metrics_by_threshold'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs metric/threshold charts to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_class_metrics_by_threshold neptune.init() with neptune.create_experiment(): log_class_metrics_by_threshold(y_test, y_test_pred[:,1]) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_pos.shape ) == 1, 'y_pred_pos needs to be 1D prediction for positive class' _exp = experiment if experiment else neptune expect_not_a_run(_exp) figs = _plot_class_metrics_by_threshold(y_true, y_pred_pos) for fig in figs: send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def save_codebook(self, weight_list, epoch, label): figure, axes = plt.subplots(1, 3, sharey=True, figsize=(20, 5)) figure.suptitle(f'Epoch {epoch}') for i, (matrix, ax) in enumerate(zip(weight_list, axes)): g = sns.heatmap(matrix, annot=True, fmt='.2f', ax=ax) g.set_title(f'{label} {i}') send_figure(figure, channel_name=label) plt.close()
def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''): """Creates and logs Precision Recall curve and Average precision score to Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs Precision Recall curve and Average precision score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_precision_recall_auc neptune.init() with neptune.create_experiment(): log_precision_recall_auc(y_test, y_test_pred) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune expect_not_a_run(_exp) avg_precision = sk_metrics.average_precision_score(y_true, y_pred[:, 1]) _exp.log_metric(prefix + 'avg_precision', avg_precision) fig, ax = plt.subplots() plt_metrics.plot_precision_recall(y_true, y_pred, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_confusion_matrix(y_true, y_pred_class, experiment=None, channel_name='metric_charts', prefix=''): """Creates a confusion matrix figure and logs it in Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Log confusion matrix to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_confusion_matrix neptune.init() with neptune.create_experiment(): log_confusion_matrix(y_test, y_test_pred[:,1]>0.5) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_class.shape ) == 1, 'y_pred_class needs to be 1D class prediction with values 0, 1' _exp = experiment if experiment else neptune expect_not_a_run(_exp) fig, ax = plt.subplots() _plot_confusion_matrix(y_true, y_pred_class, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_prediction_distribution(y_true, y_pred_pos, experiment=None, channel_name='metric_charts', prefix=''): """Generates prediction distribution plot from predictions and true labels. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Plot prediction distribution:: from neptunecontrib.monitoring.metrics import log_prediction_distribution log_prediction_distribution(y_test, y_test_pred[:, 1]) """ assert len( y_pred_pos.shape ) == 1, 'y_pred_pos needs to be 1D prediction for positive class' _exp = experiment if experiment else neptune expect_not_a_run(_exp) fig, ax = plt.subplots() _plot_prediction_distribution(y_true, y_pred_pos, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def on_epoch_end(self, *args): loader = DataLoader(self.dataset, batch_size=500, drop_last=False, shuffle=True) input, _ = next(iter(loader)) g1, g2 = self.get_gradients(input) neptune.send_text('color_grad', str(g1)) neptune.send_text('shape_grad', str(g2)) ax = sns.heatmap(torch.cat([g1, g2], dim=0), xticklabels=['color', 'shape'], yticklabels=['$m_1$', '$m_2$']) figure = ax.get_figure() send_figure(figure, channel_name=f'{self.label} grads') figure.savefig('fig.png') plt.close(figure)
def visualize_embeddings(self): embeddings = self.game.receiver.embedding.weight.detach().transpose( 1, 0) pca = PCA(n_components=2) embeddings_projected = pca.fit_transform(embeddings) np.savetxt('embs.txt', embeddings_projected) neptune.send_artifact('embs.txt') ax = sns.scatterplot(x=embeddings_projected[:, 0], y=embeddings_projected[:, 1]) for i in range(10): ax.annotate(str(i), embeddings_projected[i], size=20) sns.despine(left=True, bottom=True) plt.xlabel('First principal component') plt.ylabel('Second principal component') figure = ax.get_figure() send_figure(figure, channel_name='embeddings') figure.savefig('figx.png') plt.close(figure)
def visualize(self, dimensionality_reduction_transform): *_, last_activations = zip(*[ activations for (key, activations) in sorted(self.activations.items()) ]) activations_projected = dimensionality_reduction_transform.fit_transform( np.vstack(last_activations)) df = pd.DataFrame.from_dict({ 'color': (['blue'] * 5 + ['cyan'] * 5 + ['gray'] * 5 + ['green'] * 5 + ['magenta'] * 5), 'shape': ['box', 'sphere', 'cylinder', 'torus', 'ellipsoid'] * 5, 'x': activations_projected[:, 0], 'y': activations_projected[:, 1] }) sns.set(style="whitegrid") ax = sns.scatterplot(x="x", y="y", hue="color", style="shape", data=df, legend=False, palette=dict(blue='blue', cyan='cyan', gray='gray', green='green', magenta='magenta'), markers=('s', 'o', 'D', 'X', '^'), s=70) sns.despine(left=True, bottom=True) plt.xlabel('First principal component') plt.ylabel('Second principal component') figure = ax.get_figure() send_figure(figure, channel_name=self.label) figure.savefig('fig.png') plt.close(figure)
def log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, sensitive_attributes, favorable_label, unfavorable_label, privileged_groups, unprivileged_groups, experiment=None, prefix=''): """Creates fairness metric charts, calculates fairness classification metrics and logs them to Neptune. Class-based metrics that are logged: 'true_positive_rate_difference','false_positive_rate_difference', 'false_omission_rate_difference', 'false_discovery_rate_difference', 'error_rate_difference', 'false_positive_rate_ratio', 'false_negative_rate_ratio', 'false_omission_rate_ratio', 'false_discovery_rate_ratio', 'error_rate_ratio', 'average_odds_difference', 'disparate_impact', 'statistical_parity_difference', 'equal_opportunity_difference', 'theil_index', 'between_group_theil_index', 'between_all_groups_theil_index', 'coefficient_of_variation', 'between_group_coefficient_of_variation', 'between_all_groups_coefficient_of_variation', 'generalized_entropy_index', 'between_group_generalized_entropy_index', 'between_all_groups_generalized_entropy_index' Charts are logged to the 'metric_by_group' channel: 'confusion matrix', 'TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV', 'FDR', 'FOR', 'ACC', 'error_rate', 'selection_rate', 'power', 'precision', 'recall', 'sensitivity', 'specificity'. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1. y_pred_score (array-like, shape (n_samples)): Class predictions with values from 0 to 1. Default None. sensitive_attributes (pandas.DataFrame, shape (n_samples, k)): datafame containing only sensitive columns. favorable_label (str or int): label that is favorable, brings positive value to a person being classified. unfavorable_label (str or int): label that is unfavorable, brings positive value to a person being classified. privileged_groups (dict): dictionary with column names and list of values for those columns that belong to the privileged groups. unprivileged_groups (dict): dictionary with column names and list of values for those columns that belong to the unprivileged groups. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test. Log metrics and performance curves to Neptune:: import neptune from neptunecontrib.monitoring.fairness import log_fairness_classification_metrics neptune.init() with neptune.create_experiment(): log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, test[['race']], favorable_label='granted_parole', unfavorable_label='not_granted_parole', privileged_groups={'race':['Caucasian']}, privileged_groups={'race':['African-American','Hispanic]}, ) Check out this experiment https://ui.neptune.ai/jakub-czakon/model-fairness/e/MOD-92/logs. """ _exp = experiment if experiment else neptune expect_not_a_run(_exp) bias_info = {'favorable_label': favorable_label, 'unfavorable_label': unfavorable_label, 'protected_columns': sensitive_attributes.columns.tolist()} privileged_info = _fmt_priveleged_info(privileged_groups, unprivileged_groups) ground_truth_test = _make_dataset(sensitive_attributes, y_true, **bias_info, **privileged_info) prediction_test = _make_dataset(sensitive_attributes, y_pred_class, y_pred_score, **bias_info, **privileged_info) clf_metric = ClassificationMetric(ground_truth_test, prediction_test, **privileged_info) _log_fairness_metrics(clf_metric, _exp, prefix) fig = _plot_confusion_matrix_by_group(clf_metric, figsize=(12, 4)) plt.tight_layout() plt.close() send_figure(fig, channel_name=prefix + 'metrics_by_group') group_metrics = ['TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV', 'FDR', 'FOR', 'ACC', 'error_rate', 'selection_rate', 'power', 'precision', 'recall', 'sensitivity', 'specificity'] for metric_name in group_metrics: fig, ax = plt.subplots(figsize=(12, 8)) _plot_performance_by_group(clf_metric, metric_name, ax) send_figure(fig, experiment=_exp, channel_name=prefix + 'metrics_by_group') plt.close()