def extract_output(self, accumulator: _State) -> Dict[str, Any]: return { metric_keys.base_key('total_queries'): accumulator.total_queries, metric_keys.base_key('total_documents'): accumulator.total_documents, metric_keys.base_key('min_documents'): accumulator.min_documents, metric_keys.base_key('max_documents'): accumulator.max_documents }
def check_metric_result(got): try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) expected_values_dict = { metric_keys.base_key('english_head/logistic/[email protected]'): 1.0, metric_keys.base_key('chinese_head/logistic/[email protected]'): 1.0, } self.assertDictElementsAlmostEqual(value, expected_values_dict) except AssertionError as err: raise util.BeamAssertException(err)
def extract_output(self, accumulator: _State) -> Dict[str, Any]: if accumulator.weight_sum > 0: return { metric_keys.base_key('average_min_label_position/%s' % self._label_key): accumulator.min_pos_sum / accumulator.weight_sum } return {}
def extract_output(self, accumulator: _State) -> Dict[str, Any]: avg_dict = {} for at in self._at_vals: if accumulator.weight > 0: avg_ndcg = accumulator.ndcg[at] / accumulator.weight else: avg_ndcg = 0 avg_dict[metric_keys.base_key('ndcg@%d' % at)] = avg_ndcg return avg_dict
def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) self.assertTrue( np.isnan(value[metric_keys.base_key( '[email protected]')])) except AssertionError as err: raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) expected_values_dict = { metric_keys.base_key('[email protected]'): 0.0, } self.assertDictElementsAlmostEqual(value, expected_values_dict) except AssertionError as err: raise util.BeamAssertException(err)
def get_metric_ops(self, features_dict: types.TensorTypeMaybeDict, predictions_dict: types.TensorTypeMaybeDict, labels_dict: types.TensorTypeMaybeDict ) -> Dict[Text, Tuple[types.TensorType, types.TensorType]]: values, update_ops = self.confusion_matrix_metric_ops( features_dict, predictions_dict, labels_dict) # True positive rate is computed by confusion_matrix_metric_ops as 'recall'. # pytype: disable=unsupported-operands values['tnr'] = values['tn'] / (values['tn'] + values['fp']) values['fpr'] = values['fp'] / (values['fp'] + values['tn']) values['positive_rate'] = (values['tp'] + values['fp']) / ( values['tp'] + values['fp'] + values['tn'] + values['fn']) values['fnr'] = values['fn'] / (values['fn'] + values['tp']) values['negative_rate'] = (values['tn'] + values['fn']) / ( values['tp'] + values['fp'] + values['tn'] + values['fn']) # pytype: enable=unsupported-operands update_op = tf.group(update_ops['fn'], update_ops['tn'], update_ops['fp'], update_ops['tp']) value_op = tf.transpose( a=tf.stack([ values['fn'], values['tn'], values['fp'], values['tp'], values['precision'], values['recall'] ])) output_dict = { self._metric_key(self.matrices_key): (value_op, update_op), self._metric_key(self.thresholds_key): (tf.identity(self._thresholds), tf.no_op()), } for i, threshold in enumerate(self._thresholds): output_dict[self._metric_key( metric_keys.base_key( 'positive_rate@%.*f' % (self._key_digits, threshold)))] = (values['positive_rate'][i], update_op) output_dict[self._metric_key( metric_keys.base_key( 'true_positive_rate@%.*f' % (self._key_digits, threshold)))] = (values['recall'][i], update_op) output_dict[self._metric_key( metric_keys.base_key( 'false_positive_rate@%.*f' % (self._key_digits, threshold)))] = (values['fpr'][i], update_op) output_dict[self._metric_key( metric_keys.base_key( 'negative_rate@%.*f' % (self._key_digits, threshold)))] = (values['negative_rate'][i], update_op) output_dict[self._metric_key( metric_keys.base_key( 'true_negative_rate@%.*f' % (self._key_digits, threshold)))] = (values['tnr'][i], update_op) output_dict[self._metric_key( metric_keys.base_key( 'false_negative_rate@%.*f' % (self._key_digits, threshold)))] = (values['fnr'][i], update_op) return output_dict # pytype: disable=bad-return-type
def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) self.assertIn(metric_keys.base_key('[email protected]'), value) except AssertionError as err: raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) expected_values_dict = { metric_keys.base_key('[email protected]'): 6.0 / 7.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 0.8, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 7.0, metric_keys.base_key('[email protected]'): 2.0 / 10.0, metric_keys.base_key('[email protected]'): 3.0 / 7.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 0.5, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 4.0 / 7.0, metric_keys.base_key('[email protected]'): 5.0 / 10.0, metric_keys.base_key('[email protected]'): 3.0 / 7.0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 0.3, metric_keys.base_key('[email protected]'): 3.0 / 3.0, metric_keys.base_key('[email protected]'): 4.0 / 7.0, metric_keys.base_key('[email protected]'): 7.0 / 10.0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 1, metric_keys.base_key('[email protected]'): 1, metric_keys.base_key('[email protected]'): 1, } self.assertDictElementsWithTDistributionAlmostEqual( value, expected_values_dict) except AssertionError as err: raise util.BeamAssertException(err)
def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) expected_values_dict = { metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 3.0 / 5.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 2.0 / 5.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 2.0 / 5.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 3.0 / 5.0, metric_keys.base_key('[email protected]'): 1.0 / 5.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 0.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 5.0, metric_keys.base_key('[email protected]'): 2.0 / 2.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 4.0 / 5.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 3.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 1.0 / 2.0, metric_keys.base_key('[email protected]'): 2.0 / 3.0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 2.0 / 4.0, metric_keys.base_key('[email protected]'): 0, metric_keys.base_key('[email protected]'): 2.0 / 4.0, } self.assertDictElementsAlmostEqual(value, expected_values_dict) except AssertionError as err: raise util.BeamAssertException(err)