def extract_output(self, accumulator: _State) -> Dict[str, Any]:
return {
metric_keys.base_key('total_queries'): accumulator.total_queries,
metric_keys.base_key('total_documents'): accumulator.total_documents,
metric_keys.base_key('min_documents'): accumulator.min_documents,
metric_keys.base_key('max_documents'): accumulator.max_documents
}
def check_metric_result(got):
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
expected_values_dict = {
metric_keys.base_key('english_head/logistic/[email protected]'):
1.0,
metric_keys.base_key('chinese_head/logistic/[email protected]'):
1.0,
}
self.assertDictElementsAlmostEqual(value, expected_values_dict)
except AssertionError as err:
raise util.BeamAssertException(err)
def extract_output(self, accumulator: _State) -> Dict[str, Any]:
if accumulator.weight_sum > 0:
return {
metric_keys.base_key('average_min_label_position/%s' % self._label_key):
accumulator.min_pos_sum / accumulator.weight_sum
}
return {}
def extract_output(self, accumulator: _State) -> Dict[str, Any]:
avg_dict = {}
for at in self._at_vals:
if accumulator.weight > 0:
avg_ndcg = accumulator.ndcg[at] / accumulator.weight
else:
avg_ndcg = 0
avg_dict[metric_keys.base_key('ndcg@%d' % at)] = avg_ndcg
return avg_dict
def check_result(got): # pylint: disable=invalid-name
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
self.assertTrue(
np.isnan(value[metric_keys.base_key(
'[email protected]')]))
except AssertionError as err:
raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
expected_values_dict = {
metric_keys.base_key('[email protected]'): 0.0,
}
self.assertDictElementsAlmostEqual(value, expected_values_dict)
except AssertionError as err:
raise util.BeamAssertException(err)
def get_metric_ops(self, features_dict: types.TensorTypeMaybeDict,
predictions_dict: types.TensorTypeMaybeDict,
labels_dict: types.TensorTypeMaybeDict
) -> Dict[Text, Tuple[types.TensorType, types.TensorType]]:
values, update_ops = self.confusion_matrix_metric_ops(
features_dict, predictions_dict, labels_dict)
# True positive rate is computed by confusion_matrix_metric_ops as 'recall'.
# pytype: disable=unsupported-operands
values['tnr'] = values['tn'] / (values['tn'] + values['fp'])
values['fpr'] = values['fp'] / (values['fp'] + values['tn'])
values['positive_rate'] = (values['tp'] + values['fp']) / (
values['tp'] + values['fp'] + values['tn'] + values['fn'])
values['fnr'] = values['fn'] / (values['fn'] + values['tp'])
values['negative_rate'] = (values['tn'] + values['fn']) / (
values['tp'] + values['fp'] + values['tn'] + values['fn'])
# pytype: enable=unsupported-operands
update_op = tf.group(update_ops['fn'], update_ops['tn'], update_ops['fp'],
update_ops['tp'])
value_op = tf.transpose(
a=tf.stack([
values['fn'], values['tn'], values['fp'], values['tp'],
values['precision'], values['recall']
]))
output_dict = {
self._metric_key(self.matrices_key): (value_op, update_op),
self._metric_key(self.thresholds_key): (tf.identity(self._thresholds),
tf.no_op()),
}
for i, threshold in enumerate(self._thresholds):
output_dict[self._metric_key(
metric_keys.base_key(
'positive_rate@%.*f' %
(self._key_digits, threshold)))] = (values['positive_rate'][i],
update_op)
output_dict[self._metric_key(
metric_keys.base_key(
'true_positive_rate@%.*f' %
(self._key_digits, threshold)))] = (values['recall'][i],
update_op)
output_dict[self._metric_key(
metric_keys.base_key(
'false_positive_rate@%.*f' %
(self._key_digits, threshold)))] = (values['fpr'][i], update_op)
output_dict[self._metric_key(
metric_keys.base_key(
'negative_rate@%.*f' %
(self._key_digits, threshold)))] = (values['negative_rate'][i],
update_op)
output_dict[self._metric_key(
metric_keys.base_key(
'true_negative_rate@%.*f' %
(self._key_digits, threshold)))] = (values['tnr'][i], update_op)
output_dict[self._metric_key(
metric_keys.base_key(
'false_negative_rate@%.*f' %
(self._key_digits, threshold)))] = (values['fnr'][i], update_op)
return output_dict # pytype: disable=bad-return-type
def check_result(got): # pylint: disable=invalid-name
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
self.assertIn(metric_keys.base_key('[email protected]'), value)
except AssertionError as err:
raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
expected_values_dict = {
metric_keys.base_key('[email protected]'): 6.0 / 7.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 0.8,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 7.0,
metric_keys.base_key('[email protected]'): 2.0 / 10.0,
metric_keys.base_key('[email protected]'): 3.0 / 7.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 0.5,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 4.0 / 7.0,
metric_keys.base_key('[email protected]'): 5.0 / 10.0,
metric_keys.base_key('[email protected]'): 3.0 / 7.0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 0.3,
metric_keys.base_key('[email protected]'): 3.0 / 3.0,
metric_keys.base_key('[email protected]'): 4.0 / 7.0,
metric_keys.base_key('[email protected]'): 7.0 / 10.0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 1,
metric_keys.base_key('[email protected]'): 1,
metric_keys.base_key('[email protected]'): 1,
}
self.assertDictElementsWithTDistributionAlmostEqual(
value, expected_values_dict)
except AssertionError as err:
raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name
try:
self.assertEqual(1, len(got), 'got: %s' % got)
(slice_key, value) = got[0]
self.assertEqual((), slice_key)
expected_values_dict = {
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 3.0 / 5.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 2.0 / 5.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 2.0 / 5.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 3.0 / 5.0,
metric_keys.base_key('[email protected]'): 1.0 / 5.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 0.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 5.0,
metric_keys.base_key('[email protected]'): 2.0 / 2.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 4.0 / 5.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 3.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 1.0 / 2.0,
metric_keys.base_key('[email protected]'): 2.0 / 3.0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 2.0 / 4.0,
metric_keys.base_key('[email protected]'): 0,
metric_keys.base_key('[email protected]'): 2.0 / 4.0,
}
self.assertDictElementsAlmostEqual(value, expected_values_dict)
except AssertionError as err:
raise util.BeamAssertException(err)
