def test_use_entities_order_one_output_edge(self):
existing_edges_filter = ExistingEdgesFilter(
entities_count=3,
graph_edges=self.graph_edges,
use_entities_order=True)
filtered_values, target_index = existing_edges_filter.get_values_corresponding_to_existing_edges(
edge_ids=[2, 1, 2], mask_index=2, values=np.array([1.0, 0.5, 2.0]))
self.assertAllEqual([2.0], filtered_values)
self.assertEqual(0, target_index)
def test_target_index(self, randint_mock):
randint_mock.return_value = 1
existing_edges_filter = ExistingEdgesFilter(
entities_count=3,
graph_edges=self.graph_edges,
use_entities_order=False)
filtered_values, target_index = existing_edges_filter.get_values_corresponding_to_existing_edges(
edge_ids=[0, 0, 2], mask_index=2, values=np.array([1.0, 0.5, 2.0]))
self.assertAllEqual([1.0, 2.0], filtered_values)
self.assertEqual(1, target_index)
def __init__(self, **kwargs):
super(SoftmaxModelEvaluator, self).__init__(**kwargs)
existing_graph_edges = dataset_utils.get_existing_graph_edges(
self.dataset.data_directory)
self.existing_edges_filter = ExistingEdgesFilter(
self.dataset.entities_count, existing_graph_edges)
class SoftmaxModelEvaluator(ModelEvaluator):
def __init__(self, **kwargs):
super(SoftmaxModelEvaluator, self).__init__(**kwargs)
existing_graph_edges = dataset_utils.get_existing_graph_edges(
self.dataset.data_directory)
self.existing_edges_filter = ExistingEdgesFilter(
self.dataset.entities_count, existing_graph_edges)
def _compute_metrics_on_samples(self, list_of_batched_samples):
raw_head_metrics, raw_tail_metrics = EvaluationMetrics(
), EvaluationMetrics()
head_metrics, tail_metrics = EvaluationMetrics(), EvaluationMetrics()
for batched_samples in list_of_batched_samples:
losses_ranking = 1.0 - self.model(
batched_samples,
training=False).numpy()[:, :self.dataset.entities_count]
for sample, ranking in zip(_unbatch_samples(batched_samples),
losses_ranking):
filtered_ranking, target_index = self.existing_edges_filter.get_values_corresponding_to_existing_edges(
sample["edge_ids"], sample["mask_index"], values=ranking)
if sample["mask_index"] == 0:
raw_head_metrics.update_state(
ranking,
positive_sample_index=sample["expected_output"])
head_metrics.update_state(
filtered_ranking, positive_sample_index=target_index)
elif sample["mask_index"] == 2:
raw_tail_metrics.update_state(
ranking,
positive_sample_index=sample["expected_output"])
tail_metrics.update_state(
filtered_ranking, positive_sample_index=target_index)
else:
raise ValueError(
f"Invalid `mask_index`: {sample['mask_index']}")
raw_averaged_evaluation_metrics = EvaluationMetrics.get_average_metrics(
raw_head_metrics, raw_tail_metrics)
averaged_evaluation_metrics = EvaluationMetrics.get_average_metrics(
head_metrics, tail_metrics)
return {
"raw_metrics_head": raw_head_metrics,
"raw_metrics_tail": raw_tail_metrics,
"raw_metrics_averaged": raw_averaged_evaluation_metrics,
"metrics_head": head_metrics,
"metrics_tail": tail_metrics,
"metrics_averaged": averaged_evaluation_metrics,
}
def _compute_and_report_losses(self, list_of_batched_samples, step):
loss_values = []
for batched_samples in list_of_batched_samples:
predictions = self.model(batched_samples, training=False)
loss_values.extend(
self.loss_object.get_losses_of_samples(
true_labels=batched_samples["expected_output"],
predictions=predictions))
loss_value = tf.reduce_mean(loss_values)
tf.summary.scalar(name="losses/samples_loss",
data=loss_value,
step=step)
regularization_loss = self.loss_object.get_regularization_loss(
self.model)
tf.summary.scalar(name="losses/regularization_loss",
data=regularization_loss,
step=step)
return loss_value.numpy()
def build_model(self):
self.model(next(self.iterator_of_samples), training=False)
def evaluation_step(self, step):
list_of_batched_samples = [
next(self.iterator_of_samples),
next(self.iterator_of_samples)
]
with self.summary_writer.as_default():
metrics = self._compute_and_report_metrics(list_of_batched_samples,
step)
self._compute_and_report_losses(list_of_batched_samples, step)
self._maybe_report_learning_rate(step)
return metrics["metrics_averaged"].result()[1]
def log_metrics(self, logger):
samples_iterator = self.dataset.samples.as_numpy_iterator()
named_metrics = self._compute_metrics_on_samples(
list_of_batched_samples=samples_iterator)
_log_dict_of_metrics(logger, named_metrics)