def testCalibrationPlotAndPredictionHistogramWeighted(self): temp_eval_export_dir = self._getEvalExportDir() _, eval_export_dir = (fixed_prediction_estimator_extra_fields. simple_fixed_prediction_estimator_extra_fields( None, temp_eval_export_dir)) examples = [ # For each example, we set label to prediction + 1. self._makeExample(prediction=-10.0, label=-9.0, fixed_float=1.0, fixed_string=''), self._makeExample(prediction=-9.0, label=-8.0, fixed_float=2.0, fixed_string=''), self._makeExample(prediction=0.0000, label=1.0000, fixed_float=0.0, fixed_string=''), self._makeExample(prediction=0.00100, label=1.00100, fixed_float=1.0, fixed_string=''), self._makeExample(prediction=0.00101, label=1.00101, fixed_float=2.0, fixed_string=''), self._makeExample(prediction=0.00102, label=1.00102, fixed_float=3.0, fixed_string=''), self._makeExample(prediction=10.0, label=11.0, fixed_float=7.0, fixed_string=''), ] def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) self.assertIn(metric_keys.CALIBRATION_PLOT_MATRICES, value) buckets = value[metric_keys.CALIBRATION_PLOT_MATRICES] self.assertSequenceAlmostEqual(buckets[0], [-28.0, -25.0, 3.0]) self.assertSequenceAlmostEqual(buckets[1], [0.0, 0.0, 0.0]) self.assertSequenceAlmostEqual(buckets[11], [0.00608, 6.00608, 6.0]) self.assertSequenceAlmostEqual(buckets[10001], [70.0, 77.0, 7.0]) except AssertionError as err: raise util.BeamAssertException(err) self._runTestWithCustomCheck( examples, eval_export_dir, [ post_export_metrics.calibration_plot_and_prediction_histogram( example_weight_key='fixed_float') ], custom_plots_check=check_result)
def testSerializePlots(self): slice_key = _make_slice_key('fruit', 'apple') tfma_plots = { metric_keys.CALIBRATION_PLOT_MATRICES: np.array([ [0.0, 0.0, 0.0], [0.3, 1.0, 1.0], [0.7, 0.0, 1.0], [0.0, 0.0, 0.0], ]), metric_keys.CALIBRATION_PLOT_BOUNDARIES: np.array([0.0, 0.5, 1.0]), } expected_plot_data = """ slice_key { single_slice_keys { column: 'fruit' bytes_value: 'apple' } } plot_data { calibration_histogram_buckets { buckets { lower_threshold_inclusive: -inf upper_threshold_exclusive: 0.0 num_weighted_examples { value: 0.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.0 } } buckets { lower_threshold_inclusive: 0.0 upper_threshold_exclusive: 0.5 num_weighted_examples { value: 1.0 } total_weighted_label { value: 1.0 } total_weighted_refined_prediction { value: 0.3 } } buckets { lower_threshold_inclusive: 0.5 upper_threshold_exclusive: 1.0 num_weighted_examples { value: 1.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.7 } } buckets { lower_threshold_inclusive: 1.0 upper_threshold_exclusive: inf num_weighted_examples { value: 0.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.0 } } } } """ calibration_plot = ( post_export_metrics.calibration_plot_and_prediction_histogram()) serialized = serialization._serialize_plots((slice_key, tfma_plots), [calibration_plot]) self.assertProtoEquals( expected_plot_data, metrics_for_slice_pb2.PlotsForSlice.FromString(serialized))
def testCalibrationPlotAndPredictionHistogramUnweighted(self): temp_eval_export_dir = self._getEvalExportDir() _, eval_export_dir = ( fixed_prediction_estimator.simple_fixed_prediction_estimator( None, temp_eval_export_dir)) examples = [ # For each example, we set label to prediction + 1. # These two go in bucket 0: (-inf, 0) self._makeExample(prediction=-10.0, label=-9.0), self._makeExample(prediction=-9.0, label=-8.0), # This goes in bucket 1: [0, 0.00100) self._makeExample(prediction=0.00000, label=1.00000), # These three go in bucket 1: [0.00100, 0.00110) self._makeExample(prediction=0.00100, label=1.00100), self._makeExample(prediction=0.00101, label=1.00101), self._makeExample(prediction=0.00102, label=1.00102), # These two go in bucket 10000: [0.99990, 1.00000) self._makeExample(prediction=0.99998, label=1.99998), self._makeExample(prediction=0.99999, label=1.99999), # These four go in bucket 10001: [1.0000, +inf) self._makeExample(prediction=1.0, label=2.0), self._makeExample(prediction=8.0, label=9.0), self._makeExample(prediction=9.0, label=10.0), self._makeExample(prediction=10.0, label=11.0), ] def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) self.assertIn(metric_keys.CALIBRATION_PLOT_MATRICES, value) buckets = value[metric_keys.CALIBRATION_PLOT_MATRICES] self.assertSequenceAlmostEqual(buckets[0], [-19.0, -17.0, 2.0]) self.assertSequenceAlmostEqual(buckets[1], [0.0, 1.0, 1.0]) self.assertSequenceAlmostEqual(buckets[11], [0.00303, 3.00303, 3.0]) self.assertSequenceAlmostEqual(buckets[10000], [1.99997, 3.99997, 2.0]) self.assertSequenceAlmostEqual(buckets[10001], [28.0, 32.0, 4.0]) self.assertIn(metric_keys.CALIBRATION_PLOT_BOUNDARIES, value) boundaries = value[metric_keys.CALIBRATION_PLOT_BOUNDARIES] self.assertAlmostEqual(0.0, boundaries[0]) self.assertAlmostEqual(0.001, boundaries[10]) self.assertAlmostEqual(0.005, boundaries[50]) self.assertAlmostEqual(0.010, boundaries[100]) self.assertAlmostEqual(0.100, boundaries[1000]) self.assertAlmostEqual(0.800, boundaries[8000]) self.assertAlmostEqual(1.000, boundaries[10000]) except AssertionError as err: raise util.BeamAssertException(err) self._runTestWithCustomCheck( examples, eval_export_dir, [post_export_metrics.calibration_plot_and_prediction_histogram()], custom_plots_check=check_result)
def testCalibrationPlotSerialization(self): # Calibration plots for the model # {prediction:0.3, true_label:+}, # {prediction:0.7, true_label:-} # # These plots were generated by hand. For this test to make sense # it must actually match the kind of output the TFMA produces. tfma_plots = { metric_keys.CALIBRATION_PLOT_MATRICES: np.array([ [0.0, 0.0, 0.0], [0.3, 1.0, 1.0], [0.7, 0.0, 1.0], [0.0, 0.0, 0.0], ]), metric_keys.CALIBRATION_PLOT_BOUNDARIES: np.array([0.0, 0.5, 1.0]), } expected_plot_data = """ calibration_histogram_buckets { buckets { lower_threshold_inclusive: -inf upper_threshold_exclusive: 0.0 num_weighted_examples { value: 0.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.0 } } buckets { lower_threshold_inclusive: 0.0 upper_threshold_exclusive: 0.5 num_weighted_examples { value: 1.0 } total_weighted_label { value: 1.0 } total_weighted_refined_prediction { value: 0.3 } } buckets { lower_threshold_inclusive: 0.5 upper_threshold_exclusive: 1.0 num_weighted_examples { value: 1.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.7 } } buckets { lower_threshold_inclusive: 1.0 upper_threshold_exclusive: inf num_weighted_examples { value: 0.0 } total_weighted_label { value: 0.0 } total_weighted_refined_prediction { value: 0.0 } } } """ plot_data = metrics_for_slice_pb2.PlotData() calibration_plot = ( post_export_metrics.calibration_plot_and_prediction_histogram()) calibration_plot.populate_plots_and_pop(tfma_plots, plot_data) self.assertProtoEquals(expected_plot_data, plot_data) self.assertFalse(metric_keys.CALIBRATION_PLOT_MATRICES in tfma_plots) self.assertFalse(metric_keys.CALIBRATION_PLOT_BOUNDARIES in tfma_plots)
def main(): tf.logging.set_verbosity(tf.logging.INFO) args = parse_arguments() tfma_result = run_tfma(input_csv=args.input_csv, tfma_run_dir=args.tfma_run_dir, eval_model_base_dir=args.eval_model_dir, slice_spec=ALL_SPECS, working_dir=args.tfma_run_dir, mode=args.mode, project=args.project, setup_file=args.setup_file, add_metrics_callbacks=[ post_export_metrics.calibration_plot_and_prediction_histogram(), post_export_metrics.auc_plots()] )
def main(): tf.logging.set_verbosity(tf.logging.INFO) args = parse_arguments() tfma_result = run_tfma( input_csv=args.input_csv, tfma_run_dir=args.tfma_run_dir, eval_model_base_dir=args.eval_model_dir, slice_spec=ALL_SPECS, working_dir=args.tfma_run_dir, mode=args.mode, project=args.project, setup_file=args.setup_file, add_metrics_callbacks=[ post_export_metrics.calibration_plot_and_prediction_histogram(), post_export_metrics.auc_plots() ])
def testCalibrationPlotAndPredictionHistogramLinearClassifier(self): temp_eval_export_dir = self._getEvalExportDir() _, eval_export_dir = (linear_classifier.simple_linear_classifier( None, temp_eval_export_dir)) examples = [ self._makeExample(age=3.0, language='english', label=1.0), self._makeExample(age=3.0, language='chinese', label=0.0), self._makeExample(age=4.0, language='english', label=1.0), self._makeExample(age=5.0, language='chinese', label=0.0) ] def check_result(got): # pylint: disable=invalid-name try: self.assertEqual(1, len(got), 'got: %s' % got) (slice_key, value) = got[0] self.assertEqual((), slice_key) self.assertIn(metric_keys.CALIBRATION_PLOT_MATRICES, value) # We just check that the bucket sums look sane, since we don't know # the exact predictions of the model. # # Note that the correctness of the bucketing is tested in the other # two tests with the fixed prediction estimator. This test is more # for ensuring that this metric is compatible with the canned # Estimators, for which the prediction Tensor returned for a batch # of examples will be a N x 1 Tensor, rather than just an N element # vector. buckets = value[metric_keys.CALIBRATION_PLOT_MATRICES] bucket_sums = np.sum(buckets, axis=0) self.assertAlmostEqual(bucket_sums[1], 2.0) # label sum self.assertAlmostEqual(bucket_sums[2], 4.0) # weight sum except AssertionError as err: raise util.BeamAssertException(err) self._runTestWithCustomCheck( examples, eval_export_dir, [post_export_metrics.calibration_plot_and_prediction_histogram()], custom_plots_check=check_result)
def process_tfma(eval_result_dir, schema_file, input_csv=None, big_query_table=None, eval_model_dir=None, max_eval_rows=None, pipeline_args=None): """Runs a batch job to evaluate the eval_model against the given input. Args: eval_result_dir: A directory where the evaluation result should be written to. schema_file: A file containing a text-serialized Schema that describes the eval data. input_csv: A path to a csv file which should be the input for evaluation. This can only be set if big_query_table is None. big_query_table: A BigQuery table name specified as DATASET.TABLE which should be the input for evaluation. This can only be set if input_csv is None. eval_model_dir: A directory where the eval model is located. max_eval_rows: Number of rows to query from BigQuery. pipeline_args: additional DataflowRunner or DirectRunner args passed to the beam pipeline. Raises: ValueError: if input_csv and big_query_table are not specified correctly. """ if input_csv == big_query_table and input_csv is None: raise ValueError( 'one of --input_csv or --big_query_table should be provided.') slice_spec = [ slicer.SingleSliceSpec(), slicer.SingleSliceSpec(columns=['trip_start_hour']) ] schema = taxi.read_schema(schema_file) with beam.Pipeline(argv=pipeline_args) as pipeline: if input_csv: csv_coder = taxi.make_csv_coder(schema) raw_data = ( pipeline | 'ReadFromText' >> beam.io.ReadFromText( input_csv, skip_header_lines=1) | 'ParseCSV' >> beam.Map(csv_coder.decode)) else: assert big_query_table query = taxi.make_sql(big_query_table, max_eval_rows, for_eval=True) raw_feature_spec = taxi.get_raw_feature_spec(schema) raw_data = ( pipeline | 'ReadBigQuery' >> beam.io.Read( beam.io.BigQuerySource(query=query, use_standard_sql=True)) | 'CleanData' >> beam.Map(lambda x: (taxi.clean_raw_data_dict(x, raw_feature_spec)))) # Examples must be in clean tf-example format. coder = taxi.make_proto_coder(schema) _ = ( raw_data | 'ToSerializedTFExample' >> beam.Map(coder.encode) | 'EvaluateAndWriteResults' >> tfma.EvaluateAndWriteResults( eval_saved_model_path=eval_model_dir, slice_spec=slice_spec, add_metrics_callbacks=[ post_export_metrics.calibration_plot_and_prediction_histogram(), post_export_metrics.auc_plots() ], output_path=eval_result_dir))