def test_use_entities_order_one_output_edge(self): existing_edges_filter = ExistingEdgesFilter( entities_count=3, graph_edges=self.graph_edges, use_entities_order=True) filtered_values, target_index = existing_edges_filter.get_values_corresponding_to_existing_edges( edge_ids=[2, 1, 2], mask_index=2, values=np.array([1.0, 0.5, 2.0])) self.assertAllEqual([2.0], filtered_values) self.assertEqual(0, target_index)
def test_target_index(self, randint_mock): randint_mock.return_value = 1 existing_edges_filter = ExistingEdgesFilter( entities_count=3, graph_edges=self.graph_edges, use_entities_order=False) filtered_values, target_index = existing_edges_filter.get_values_corresponding_to_existing_edges( edge_ids=[0, 0, 2], mask_index=2, values=np.array([1.0, 0.5, 2.0])) self.assertAllEqual([1.0, 2.0], filtered_values) self.assertEqual(1, target_index)
def __init__(self, **kwargs): super(SoftmaxModelEvaluator, self).__init__(**kwargs) existing_graph_edges = dataset_utils.get_existing_graph_edges( self.dataset.data_directory) self.existing_edges_filter = ExistingEdgesFilter( self.dataset.entities_count, existing_graph_edges)
class SoftmaxModelEvaluator(ModelEvaluator): def __init__(self, **kwargs): super(SoftmaxModelEvaluator, self).__init__(**kwargs) existing_graph_edges = dataset_utils.get_existing_graph_edges( self.dataset.data_directory) self.existing_edges_filter = ExistingEdgesFilter( self.dataset.entities_count, existing_graph_edges) def _compute_metrics_on_samples(self, list_of_batched_samples): raw_head_metrics, raw_tail_metrics = EvaluationMetrics( ), EvaluationMetrics() head_metrics, tail_metrics = EvaluationMetrics(), EvaluationMetrics() for batched_samples in list_of_batched_samples: losses_ranking = 1.0 - self.model( batched_samples, training=False).numpy()[:, :self.dataset.entities_count] for sample, ranking in zip(_unbatch_samples(batched_samples), losses_ranking): filtered_ranking, target_index = self.existing_edges_filter.get_values_corresponding_to_existing_edges( sample["edge_ids"], sample["mask_index"], values=ranking) if sample["mask_index"] == 0: raw_head_metrics.update_state( ranking, positive_sample_index=sample["expected_output"]) head_metrics.update_state( filtered_ranking, positive_sample_index=target_index) elif sample["mask_index"] == 2: raw_tail_metrics.update_state( ranking, positive_sample_index=sample["expected_output"]) tail_metrics.update_state( filtered_ranking, positive_sample_index=target_index) else: raise ValueError( f"Invalid `mask_index`: {sample['mask_index']}") raw_averaged_evaluation_metrics = EvaluationMetrics.get_average_metrics( raw_head_metrics, raw_tail_metrics) averaged_evaluation_metrics = EvaluationMetrics.get_average_metrics( head_metrics, tail_metrics) return { "raw_metrics_head": raw_head_metrics, "raw_metrics_tail": raw_tail_metrics, "raw_metrics_averaged": raw_averaged_evaluation_metrics, "metrics_head": head_metrics, "metrics_tail": tail_metrics, "metrics_averaged": averaged_evaluation_metrics, } def _compute_and_report_losses(self, list_of_batched_samples, step): loss_values = [] for batched_samples in list_of_batched_samples: predictions = self.model(batched_samples, training=False) loss_values.extend( self.loss_object.get_losses_of_samples( true_labels=batched_samples["expected_output"], predictions=predictions)) loss_value = tf.reduce_mean(loss_values) tf.summary.scalar(name="losses/samples_loss", data=loss_value, step=step) regularization_loss = self.loss_object.get_regularization_loss( self.model) tf.summary.scalar(name="losses/regularization_loss", data=regularization_loss, step=step) return loss_value.numpy() def build_model(self): self.model(next(self.iterator_of_samples), training=False) def evaluation_step(self, step): list_of_batched_samples = [ next(self.iterator_of_samples), next(self.iterator_of_samples) ] with self.summary_writer.as_default(): metrics = self._compute_and_report_metrics(list_of_batched_samples, step) self._compute_and_report_losses(list_of_batched_samples, step) self._maybe_report_learning_rate(step) return metrics["metrics_averaged"].result()[1] def log_metrics(self, logger): samples_iterator = self.dataset.samples.as_numpy_iterator() named_metrics = self._compute_metrics_on_samples( list_of_batched_samples=samples_iterator) _log_dict_of_metrics(logger, named_metrics)