def testCalibratedLinearClassifier(self, feature_names, output_calibration, use_bias, auc): self._ResetAllBackends() feature_columns = [ feature_column for feature_column in self.heart_feature_columns if feature_column.name in feature_names ] feature_configs = [ feature_config for feature_config in self.heart_feature_configs if feature_config.name in feature_names ] model_config = configs.CalibratedLinearConfig( use_bias=use_bias, regularizer_configs=[ configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4), ], output_calibration=output_calibration, feature_configs=feature_configs) estimator = estimators.CannedClassifier( feature_columns=feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1), optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=200)) results = estimator.evaluate(input_fn=self._GetHeartTestInputFn()) logging.info('Calibrated linear classifier results:') logging.info(results) self.assertGreater(results['auc'], auc)
def testCalibratedLinearEstimator(self, feature_names, output_calibration, use_bias, average_loss): self._ResetAllBackends() feature_columns = [ feature_column for feature_column in self.boston_feature_columns if feature_column.name in feature_names ] feature_configs = [ feature_config for feature_config in self.boston_feature_configs if feature_config.name in feature_names ] model_config = configs.CalibratedLinearConfig( use_bias=use_bias, regularizer_configs=[ configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4), ], output_calibration=output_calibration, feature_configs=feature_configs) estimator = estimators.CannedEstimator( head=regression_head.RegressionHead(), feature_columns=feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetBostonTrainInputFn(num_epochs=1), optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200)) results = estimator.evaluate(input_fn=self._GetBostonTestInputFn()) logging.info('Calibrated linear regressor results:') logging.info(results) self.assertLess(results['average_loss'], average_loss)
def testLinearH5FormatSaveLoad(self): model_config = configs.CalibratedLinearConfig( feature_configs=copy.deepcopy(feature_configs), regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-4), configs.RegularizerConfig('torsion', l2=1e-3), ], use_bias=True, output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=6, output_initialization=[0.0, 1.0]) model = premade.CalibratedLinear(model_config) # Compile and fit model. model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(0.1)) model.fit(fake_data['train_xs'], fake_data['train_ys']) # Save model using H5 format. with tempfile.NamedTemporaryFile(suffix='.h5') as f: tf.keras.models.save_model(model, f.name) loaded_model = tf.keras.models.load_model( f.name, custom_objects=premade.get_custom_objects()) self.assertAllClose( model.predict(fake_data['eval_xs']), loaded_model.predict(fake_data['eval_xs']))
def testCalibratedModelInfo(self, model_type, parameterization, output_calibration): self._ResetAllBackends() if model_type == 'linear': model_config = configs.CalibratedLinearConfig( feature_configs=self.heart_feature_configs, output_calibration=output_calibration, ) else: feature_configs = copy.deepcopy(self.heart_feature_configs) if parameterization == 'kronecker_factored': # RTL Layer only supports monotonicity and bound constraints. for feature_config in feature_configs: feature_config.lattice_size = 2 feature_config.unimodality = 'none' feature_config.reflects_trust_in = None feature_config.dominates = None feature_config.regularizer_configs = None model_config = configs.CalibratedLatticeConfig( feature_configs=feature_configs, parameterization=parameterization, output_calibration=output_calibration, ) estimator = estimators.CannedClassifier( feature_columns=self.heart_feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1), prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5), optimizer=tf.keras.optimizers.Adam(0.01), prefitting_optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20)) # Serving input fn is used to create saved models. serving_input_fn = ( tf.estimator.export.build_parsing_serving_input_receiver_fn( feature_spec=fc.make_parse_example_spec( self.heart_feature_columns))) saved_model_path = estimator.export_saved_model( estimator.model_dir, serving_input_fn) logging.info('Model exported to %s', saved_model_path) model = estimators.get_model_graph(saved_model_path) expected_num_nodes = ( 2 * len(self.heart_feature_columns) + # Input features and calibration 1 + # Linear or lattice layer int(output_calibration)) # Output calibration self.assertLen(model.nodes, expected_num_nodes)
def testLinearFromConfig(self): model_config = configs.CalibratedLinearConfig( feature_configs=copy.deepcopy(feature_configs), regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-4), configs.RegularizerConfig('torsion', l2=1e-3), ], use_bias=True, output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=6, output_initialization=[0.0, 1.0]) model = premade.CalibratedLinear(model_config) loaded_model = premade.CalibratedLinear.from_config(model.get_config()) self.assertEqual( json.dumps(model.get_config(), sort_keys=True, cls=self.Encoder), json.dumps(loaded_model.get_config(), sort_keys=True, cls=self.Encoder))
def testCalibratedModelInfo(self, model_type, output_calibration): self._ResetAllBackends() if model_type == 'linear': model_config = configs.CalibratedLinearConfig( feature_configs=self.heart_feature_configs, output_calibration=output_calibration, ) else: model_config = configs.CalibratedLatticeConfig( feature_configs=self.heart_feature_configs, output_calibration=output_calibration, ) estimator = estimators.CannedClassifier( feature_columns=self.heart_feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1), prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5), optimizer=tf.keras.optimizers.Adam(0.01), prefitting_optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20)) # Serving input fn is used to create saved models. serving_input_fn = ( tf.estimator.export.build_parsing_serving_input_receiver_fn( feature_spec=fc.make_parse_example_spec( self.heart_feature_columns))) saved_model_path = estimator.export_saved_model( estimator.model_dir, serving_input_fn) logging.info('Model exported to %s', saved_model_path) model = estimators.get_model_graph(saved_model_path) expected_num_nodes = ( 2 * len(self.heart_feature_columns) + # Input features and calibration 1 + # Linear or lattice layer int(output_calibration)) # Output calibration self.assertLen(model.nodes, expected_num_nodes)
def test_from_config(self): feature_configs = [ configs.FeatureConfig( name='feature_a', pwl_calibration_input_keypoints='quantiles', pwl_calibration_num_keypoints=8, monotonicity=1, pwl_calibration_clip_max=100, ), configs.FeatureConfig( name='feature_b', lattice_size=3, unimodality='valley', pwl_calibration_input_keypoints='uniform', pwl_calibration_num_keypoints=5, pwl_calibration_clip_min=130, pwl_calibration_convexity='convex', regularizer_configs=[ configs.RegularizerConfig(name='calib_hesian', l2=3e-3), ], ), configs.FeatureConfig( name='feature_c', pwl_calibration_input_keypoints=[0.0, 0.5, 1.0], reflects_trust_in=[ configs.TrustConfig(feature_name='feature_a'), configs.TrustConfig(feature_name='feature_b', direction=-1), ], dominates=[ configs.DominanceConfig( feature_name='feature_d', dominance_type='monotonic'), ], ), configs.FeatureConfig( name='feature_d', num_buckets=3, vocabulary_list=['a', 'b', 'c'], default_value=-1, ), ] # First we test CalibratedLatticeEnsembleConfig model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=feature_configs, lattices=[['feature_a', 'feature_b'], ['feature_c', 'feature_d']], separate_calibrators=True, regularizer_configs=[ configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=5, output_initialization=[0.0, 1.0]) model_config_copy = configs.CalibratedLatticeEnsembleConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config()) # Next we test CalibratedLatticeConfig model_config = configs.CalibratedLatticeConfig( feature_configs=feature_configs, regularizer_configs=[ configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=8, output_initialization='quantiles') model_config_copy = configs.CalibratedLatticeConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config()) # Last we test CalibratedLinearConfig model_config = configs.CalibratedLinearConfig( feature_configs=feature_configs, regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-4), ], use_bias=True, output_min=0.0, output_max=None, output_calibration=True, output_initialization='uniform') model_config_copy = configs.CalibratedLinearConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config())