def testVerifyConfig(self): unspecified_model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(unspecified_feature_configs), lattices='random', num_lattices=3, lattice_rank=2, separate_calibrators=True, output_initialization=[-1.0, 1.0]) with self.assertRaisesRegex( ValueError, 'Lattices are not fully specified for ensemble config.'): premade_lib.verify_config(unspecified_model_config) premade_lib.set_random_lattice_ensemble(unspecified_model_config) with self.assertRaisesRegex( ValueError, 'Element 0 for list/tuple 0 for feature categorical monotonicity is ' 'not an index: 0.0'): premade_lib.verify_config(unspecified_model_config) fixed_feature_configs = copy.deepcopy(unspecified_feature_configs) premade_lib.set_categorical_monotonicities(fixed_feature_configs) unspecified_model_config.feature_configs = fixed_feature_configs premade_lib.verify_config(unspecified_model_config) specified_model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(specified_feature_configs), lattices=[['numerical_1', 'categorical'], ['numerical_2', 'categorical']], num_lattices=2, lattice_rank=2, separate_calibrators=True, output_initialization=[-1.0, 1.0]) premade_lib.verify_config(specified_model_config)
def testSetRandomLattices(self): random_model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(unspecified_feature_configs), lattices='random', num_lattices=3, lattice_rank=2, separate_calibrators=True, output_initialization=[-1.0, 1.0]) premade_lib.set_random_lattice_ensemble(random_model_config) self.assertLen(random_model_config.lattices, 3) self.assertListEqual( [2, 2, 2], [len(lattice) for lattice in random_model_config.lattices]) specified_model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(specified_feature_configs), lattices=[['numerical_1', 'categorical'], ['numerical_2', 'categorical']], num_lattices=2, lattice_rank=2, separate_calibrators=True, output_initialization=[-1.0, 1.0]) with self.assertRaisesRegex( ValueError, 'model_config.lattices must be set to \'random\'.'): premade_lib.set_random_lattice_ensemble(specified_model_config)
def testLatticeEnsembleH5FormatSaveLoad(self): model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(feature_configs), lattices=[['numerical_1', 'categorical'], ['numerical_2', 'categorical']], num_lattices=2, lattice_rank=2, separate_calibrators=True, regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-3), configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=-1.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=5, output_initialization=[-1.0, 1.0]) model = premade.CalibratedLatticeEnsemble(model_config) # Compile and fit model. model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(0.1)) model.fit(fake_data['train_xs'], fake_data['train_ys']) # Save model using H5 format. with tempfile.NamedTemporaryFile(suffix='.h5') as f: tf.keras.models.save_model(model, f.name) loaded_model = tf.keras.models.load_model( f.name, custom_objects=premade.get_custom_objects()) self.assertAllClose(model.predict(fake_data['eval_xs']), loaded_model.predict(fake_data['eval_xs']))
def testLatticeEnsembleFromConfig(self): model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(feature_configs), lattices=[['numerical_1', 'categorical'], ['numerical_2', 'categorical']], num_lattices=2, lattice_rank=2, separate_calibrators=True, regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-3), configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=-1.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=5, output_initialization=[-1.0, 1.0]) model = premade.CalibratedLatticeEnsemble(model_config) loaded_model = premade.CalibratedLatticeEnsemble.from_config( model.get_config(), custom_objects=premade.get_custom_objects()) self.assertEqual( json.dumps(model.get_config(), sort_keys=True, cls=self.Encoder), json.dumps(loaded_model.get_config(), sort_keys=True, cls=self.Encoder))
def testCalibratedLatticeEnsembleModelInfo(self, lattices, num_lattices, lattice_rank, parameterization, separate_calibrators, output_calibration): self._ResetAllBackends() feature_configs = copy.deepcopy(self.heart_feature_configs) if lattices == 'rtl_layer' or parameterization == 'kronecker_factored': # RTL Layer only supports monotonicity and bound constraints. for feature_config in feature_configs: feature_config.lattice_size = 2 feature_config.unimodality = 'none' feature_config.reflects_trust_in = None feature_config.dominates = None feature_config.regularizer_configs = None model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=feature_configs, lattices=lattices, num_lattices=num_lattices, lattice_rank=lattice_rank, parameterization=parameterization, separate_calibrators=separate_calibrators, output_calibration=output_calibration, ) estimator = estimators.CannedClassifier( feature_columns=self.heart_feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1), prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5), optimizer=tf.keras.optimizers.Adam(0.01), prefitting_optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20)) # Serving input fn is used to create saved models. serving_input_fn = ( tf.estimator.export.build_parsing_serving_input_receiver_fn( feature_spec=fc.make_parse_example_spec( self.heart_feature_columns))) saved_model_path = estimator.export_saved_model( estimator.model_dir, serving_input_fn) logging.info('Model exported to %s', saved_model_path) model = estimators.get_model_graph(saved_model_path) expected_num_nodes = ( len(self.heart_feature_columns) + # Input features num_lattices + # One lattice per submodel 1 + # Averaging submodels int(output_calibration)) # Output calibration if separate_calibrators: expected_num_nodes += num_lattices * lattice_rank else: expected_num_nodes += len(self.heart_feature_columns) self.assertLen(model.nodes, expected_num_nodes)
def testCalibratedLatticeEnsembleCrystals(self): # Construct model. self._ResetAllBackends() model_config = configs.CalibratedLatticeEnsembleConfig( regularizer_configs=[ configs.RegularizerConfig(name='torsion', l2=1e-4), configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4), ], feature_configs=self.heart_feature_configs, lattices='crystals', num_lattices=6, lattice_rank=5, separate_calibrators=True, output_calibration=False, output_min=self.heart_min_label, output_max=self.heart_max_label - self.numerical_error_epsilon, output_initialization=[self.heart_min_label, self.heart_max_label], ) # Perform prefitting steps. prefitting_model_config = premade_lib.construct_prefitting_model_config( model_config) prefitting_model = premade.CalibratedLatticeEnsemble( prefitting_model_config) prefitting_model.compile( loss=tf.keras.losses.BinaryCrossentropy(), optimizer=tf.keras.optimizers.Adam(0.01)) prefitting_model.fit( self.heart_train_x, self.heart_train_y, batch_size=100, epochs=50, verbose=False) premade_lib.set_crystals_lattice_ensemble(model_config, prefitting_model_config, prefitting_model) # Construct and train final model model = premade.CalibratedLatticeEnsemble(model_config) model.compile( loss=tf.keras.losses.BinaryCrossentropy(), metrics=tf.keras.metrics.AUC(), optimizer=tf.keras.optimizers.Adam(0.01)) model.fit( self.heart_train_x, self.heart_train_y, batch_size=100, epochs=200, verbose=False) results = model.evaluate( self.heart_test_x, self.heart_test_y, verbose=False) logging.info('Calibrated lattice ensemble classifier results:') logging.info(results) self.assertGreater(results[1], 0.85)
def testCalibratedLatticeEnsembleRegressor(self, feature_names, lattices, num_lattices, lattice_rank, separate_calibrators, output_calibration, average_loss): self._ResetAllBackends() feature_columns = [ feature_column for feature_column in self.boston_feature_columns if feature_column.name in feature_names ] feature_configs = [ feature_config for feature_config in self.boston_feature_configs if feature_config.name in feature_names ] if lattices == 'rtl_layer': # RTL Layer only supports monotonicity and bound constraints. feature_configs = copy.deepcopy(feature_configs) for feature_config in feature_configs: feature_config.lattice_size = 2 feature_config.unimodality = 'none' feature_config.reflects_trust_in = None feature_config.dominates = None feature_config.regularizer_configs = None model_config = configs.CalibratedLatticeEnsembleConfig( regularizer_configs=[ configs.RegularizerConfig(name='torsion', l2=1e-5), configs.RegularizerConfig(name='output_calib_hessian', l2=1e-5), ], feature_configs=feature_configs, lattices=lattices, num_lattices=num_lattices, lattice_rank=lattice_rank, separate_calibrators=separate_calibrators, output_calibration=output_calibration, ) estimator = estimators.CannedRegressor( feature_columns=feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetBostonTrainInputFn( num_epochs=1), prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50), optimizer=tf.keras.optimizers.Adam(0.05), prefitting_optimizer=tf.keras.optimizers.Adam(0.05)) estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200)) results = estimator.evaluate(input_fn=self._GetBostonTestInputFn()) logging.info('Calibrated lattice ensemble regressor results:') logging.info(results) self.assertLess(results['average_loss'], average_loss)
def testCalibratedLatticeEnsembleRTL(self, interpolation, parameterization, num_terms, expected_minimum_auc): # Construct model. self._ResetAllBackends() rtl_feature_configs = copy.deepcopy(self.heart_feature_configs) for feature_config in rtl_feature_configs: feature_config.lattice_size = 2 feature_config.unimodality = 'none' feature_config.reflects_trust_in = None feature_config.dominates = None feature_config.regularizer_configs = None model_config = configs.CalibratedLatticeEnsembleConfig( regularizer_configs=[ configs.RegularizerConfig(name='torsion', l2=1e-4), configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4), ], feature_configs=rtl_feature_configs, lattices='rtl_layer', num_lattices=6, lattice_rank=5, interpolation=interpolation, parameterization=parameterization, num_terms=num_terms, separate_calibrators=True, output_calibration=False, output_min=self.heart_min_label, output_max=self.heart_max_label - self.numerical_error_epsilon, output_initialization=[self.heart_min_label, self.heart_max_label], ) # We must remove all regularization if using 'kronecker_factored'. if parameterization == 'kronecker_factored': model_config.regularizer_configs = None # Construct and train final model model = premade.CalibratedLatticeEnsemble(model_config) model.compile( loss=tf.keras.losses.BinaryCrossentropy(), metrics=tf.keras.metrics.AUC(), optimizer=tf.keras.optimizers.Adam(0.01)) model.fit( self.heart_train_x, self.heart_train_y, batch_size=100, epochs=200, verbose=False) results = model.evaluate( self.heart_test_x, self.heart_test_y, verbose=False) logging.info('Calibrated lattice ensemble rtl classifier results:') logging.info(results) self.assertGreater(results[1], expected_minimum_auc)
def testCalibratedLatticeEnsembleFix2dConstraintViolations( self, feature_names, lattices, num_lattices, lattice_rank, expected_lattices): self._ResetAllBackends() feature_columns = [ feature_column for feature_column in self.boston_feature_columns if feature_column.name in feature_names ] feature_configs = [ feature_config for feature_config in self.boston_feature_configs if feature_config.name in feature_names ] model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=feature_configs, lattices=lattices, num_lattices=num_lattices, lattice_rank=lattice_rank, ) estimator = estimators.CannedRegressor( feature_columns=feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetBostonTrainInputFn( num_epochs=1), prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50), optimizer=tf.keras.optimizers.Adam(0.05), prefitting_optimizer=tf.keras.optimizers.Adam(0.05)) estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200)) # Serving input fn is used to create saved models. serving_input_fn = ( tf.estimator.export.build_parsing_serving_input_receiver_fn( feature_spec=fc.make_parse_example_spec(feature_columns))) saved_model_path = estimator.export_saved_model( estimator.model_dir, serving_input_fn) logging.info('Model exported to %s', saved_model_path) model = estimators.get_model_graph(saved_model_path) lattices = [] for node in model.nodes: if isinstance(node, model_info.LatticeNode): lattices.append([ input_node.input_node.name for input_node in node.input_nodes ]) self.assertLen(lattices, len(expected_lattices)) for lattice, expected_lattice in zip(lattices, expected_lattices): self.assertCountEqual(lattice, expected_lattice)
def testCalibratedLatticeEnsembleModelInfo(self, num_lattices, lattice_rank, separate_calibrators, output_calibration): self._ResetAllBackends() model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=self.heart_feature_configs, num_lattices=num_lattices, lattice_rank=lattice_rank, separate_calibrators=separate_calibrators, output_calibration=output_calibration, ) estimator = estimators.CannedClassifier( feature_columns=self.heart_feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1), prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5), optimizer=tf.keras.optimizers.Adam(0.01), prefitting_optimizer=tf.keras.optimizers.Adam(0.01)) estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20)) # Serving input fn is used to create saved models. serving_input_fn = ( tf.estimator.export.build_parsing_serving_input_receiver_fn( feature_spec=fc.make_parse_example_spec( self.heart_feature_columns))) saved_model_path = estimator.export_saved_model( estimator.model_dir, serving_input_fn) logging.info('Model exported to %s', saved_model_path) model = estimators.get_model_graph(saved_model_path) expected_num_nodes = ( len(self.heart_feature_columns) + # Input features num_lattices + # One lattice per submodel 1 + # Averaging submodels int(output_calibration)) # Output calibration if separate_calibrators: expected_num_nodes += num_lattices * lattice_rank else: expected_num_nodes += len(self.heart_feature_columns) self.assertLen(model.nodes, expected_num_nodes)
def testCalibratedLatticeEnsembleRegressor(self, feature_names, lattices, num_lattices, lattice_rank, separate_calibrators, output_calibration, average_loss): self._ResetAllBackends() feature_columns = [ feature_column for feature_column in self.boston_feature_columns if feature_column.name in feature_names ] feature_configs = [ feature_config for feature_config in self.boston_feature_configs if feature_config.name in feature_names ] model_config = configs.CalibratedLatticeEnsembleConfig( regularizer_configs=[ configs.RegularizerConfig(name='torsion', l2=1e-5), configs.RegularizerConfig(name='output_calib_hessian', l2=1e-5), ], feature_configs=feature_configs, lattices=lattices, num_lattices=num_lattices, lattice_rank=lattice_rank, separate_calibrators=separate_calibrators, output_calibration=output_calibration, ) estimator = estimators.CannedRegressor( feature_columns=feature_columns, model_config=model_config, feature_analysis_input_fn=self._GetBostonTrainInputFn( num_epochs=1), prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50), optimizer=tf.keras.optimizers.Adam(0.05), prefitting_optimizer=tf.keras.optimizers.Adam(0.05)) estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200)) results = estimator.evaluate(input_fn=self._GetBostonTestInputFn()) logging.info('Calibrated lattice ensemble regressor results:') logging.info(results) self.assertLess(results['average_loss'], average_loss)
def testLatticeEnsembleRTLH5FormatSaveLoad(self, parameterization, num_terms): rtl_feature_configs = copy.deepcopy(feature_configs) for feature_config in rtl_feature_configs: feature_config.lattice_size = 2 feature_config.unimodality = 'none' feature_config.reflects_trust_in = None feature_config.dominates = None feature_config.regularizer_configs = None model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=copy.deepcopy(rtl_feature_configs), lattices='rtl_layer', num_lattices=2, lattice_rank=2, parameterization=parameterization, num_terms=num_terms, separate_calibrators=True, regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-3), configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=-1.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=5, output_initialization=[-1.0, 1.0]) if parameterization == 'kronecker_factored': model_config.regularizer_configs = None model = premade.CalibratedLatticeEnsemble(model_config) # Compile and fit model. model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(0.1)) model.fit(fake_data['train_xs'], fake_data['train_ys']) # Save model using H5 format. with tempfile.NamedTemporaryFile(suffix='.h5') as f: tf.keras.models.save_model(model, f.name) loaded_model = tf.keras.models.load_model( f.name, custom_objects=premade.get_custom_objects()) self.assertAllClose( model.predict(fake_data['eval_xs']), loaded_model.predict(fake_data['eval_xs']))
def test_from_config(self): feature_configs = [ configs.FeatureConfig( name='feature_a', pwl_calibration_input_keypoints='quantiles', pwl_calibration_num_keypoints=8, monotonicity=1, pwl_calibration_clip_max=100, ), configs.FeatureConfig( name='feature_b', lattice_size=3, unimodality='valley', pwl_calibration_input_keypoints='uniform', pwl_calibration_num_keypoints=5, pwl_calibration_clip_min=130, pwl_calibration_convexity='convex', regularizer_configs=[ configs.RegularizerConfig(name='calib_hesian', l2=3e-3), ], ), configs.FeatureConfig( name='feature_c', pwl_calibration_input_keypoints=[0.0, 0.5, 1.0], reflects_trust_in=[ configs.TrustConfig(feature_name='feature_a'), configs.TrustConfig(feature_name='feature_b', direction=-1), ], dominates=[ configs.DominanceConfig( feature_name='feature_d', dominance_type='monotonic'), ], ), configs.FeatureConfig( name='feature_d', num_buckets=3, vocabulary_list=['a', 'b', 'c'], default_value=-1, ), ] # First we test CalibratedLatticeEnsembleConfig model_config = configs.CalibratedLatticeEnsembleConfig( feature_configs=feature_configs, lattices=[['feature_a', 'feature_b'], ['feature_c', 'feature_d']], separate_calibrators=True, regularizer_configs=[ configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=5, output_initialization=[0.0, 1.0]) model_config_copy = configs.CalibratedLatticeEnsembleConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config()) # Next we test CalibratedLatticeConfig model_config = configs.CalibratedLatticeConfig( feature_configs=feature_configs, regularizer_configs=[ configs.RegularizerConfig('torsion', l2=1e-4), ], output_min=0.0, output_max=1.0, output_calibration=True, output_calibration_num_keypoints=8, output_initialization='quantiles') model_config_copy = configs.CalibratedLatticeConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config()) # Last we test CalibratedLinearConfig model_config = configs.CalibratedLinearConfig( feature_configs=feature_configs, regularizer_configs=[ configs.RegularizerConfig('calib_hessian', l2=1e-4), ], use_bias=True, output_min=0.0, output_max=None, output_calibration=True, output_initialization='uniform') model_config_copy = configs.CalibratedLinearConfig.from_config( model_config.get_config(), tfl_custom_objects) self.assertDictEqual(model_config.get_config(), model_config_copy.get_config())