def test_compute(): rmse = RootMeanSquaredError() y_pred = torch.Tensor([[2.0], [-2.0]]) y = torch.zeros(2) rmse.update((y_pred, y)) assert rmse.compute() == 2.0 rmse.reset() y_pred = torch.Tensor([[3.0], [-3.0]]) y = torch.zeros(2) rmse.update((y_pred, y)) assert rmse.compute() == 3.0
def test_compute(n_times, test_data): rmse = RootMeanSquaredError() y_pred, y, batch_size = test_data rmse.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size rmse.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) else: rmse.update((y_pred, y)) np_y = y.numpy().ravel() np_y_pred = y_pred.numpy().ravel() np_res = np.sqrt(np.power((np_y - np_y_pred), 2.0).sum() / np_y.shape[0]) res = rmse.compute() assert isinstance(res, float) assert pytest.approx(res) == np_res
class LocalMetrics(ignite.metrics.Metric): METRICS = ( 'rmse', 'pearson', 'per_model_pearson', ) FIGURES = ( 'hist', ) def __init__(self, column, title=None, metrics=None, figures=None, output_transform=lambda x: x): self.column = column self.title = title if title is not None else '' self.metrics = set(metrics if metrics is not None else LocalMetrics.METRICS) self.figures = set(figures if figures is not None else LocalMetrics.FIGURES) self._rmse = RootMeanSquaredError() self._pearson = PearsonR() self._per_model_pearson = Mean() self._hist = ScoreHistogram(title=title) super(LocalMetrics, self).__init__(output_transform=output_transform) def reset(self): self._rmse.reset() self._pearson.reset() self._per_model_pearson.reset() self._hist.reset() def update(self, batch: DecoyBatch): # Skip native structures and ignore residues that don't have a ground-truth score non_native = np.repeat(np.char.not_equal(batch.decoy_name, 'native'), repeats=batch.num_nodes_by_graph.cpu().numpy()) has_score = torch.isfinite(batch.lddt).cpu().numpy() valid_scores = np.logical_and(non_native, has_score) # Used to uniquely identify a (protein, model) pair without using their str names target_model_id = batch.node_index_by_graph[valid_scores].cpu().numpy() node_preds = batch.node_features[valid_scores, self.column].detach().cpu().numpy() node_targets = batch.lddt[valid_scores].detach().cpu().numpy() # Streaming metrics on local scores (they expect torch tensors, not numpy arrays) self._rmse.update((torch.from_numpy(node_preds), torch.from_numpy(node_targets))) self._pearson.update((torch.from_numpy(node_preds), torch.from_numpy(node_targets))) # Per model metrics: pandas is the easiest way to get a groupby. grouped = pd.DataFrame({ 'target_model': target_model_id, 'preds': node_preds, 'true': node_targets }).groupby('target_model') per_model_pearsons = grouped.apply(lambda df: pearson(df['preds'], df['true'])) self._per_model_pearson.update(torch.from_numpy(per_model_pearsons.values)) self._hist.update(node_preds, node_targets) def compute(self): metrics = {} figures = {} if 'rmse' in self.metrics: metrics['rmse'] = self._rmse.compute() if 'pearson' in self.metrics: metrics['pearson'] = self._pearson.compute() if 'per_model_pearson' in self.metrics: metrics['per_model_pearson'] = self._per_model_pearson.compute() if 'hist' in self.figures: extra_title = [] if 'pearson' in self.metrics: extra_title.append(f'$R$ {metrics["pearson"]:.3f}') if 'per_model_pearson' in self.metrics: extra_title.append(f'$R_\\mathrm{{model}}$ {metrics["per_model_pearson"]:.3f}') figures['hist'] = self._hist.compute('\n'.join(extra_title)) return {'metrics': metrics, 'figures': figures} def completed(self, engine, prefix): result = self.compute() for name, metric in result['metrics'].items(): engine.state.metrics[prefix + '/' + name] = metric for name, fig in result['figures'].items(): engine.state.figures[prefix + '/' + name] = fig