def testVerifyConfig(self):
unspecified_model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(unspecified_feature_configs),
lattices='random',
num_lattices=3,
lattice_rank=2,
separate_calibrators=True,
output_initialization=[-1.0, 1.0])
with self.assertRaisesRegex(
ValueError, 'Lattices are not fully specified for ensemble config.'):
premade_lib.verify_config(unspecified_model_config)
premade_lib.set_random_lattice_ensemble(unspecified_model_config)
with self.assertRaisesRegex(
ValueError,
'Element 0 for list/tuple 0 for feature categorical monotonicity is '
'not an index: 0.0'):
premade_lib.verify_config(unspecified_model_config)
fixed_feature_configs = copy.deepcopy(unspecified_feature_configs)
premade_lib.set_categorical_monotonicities(fixed_feature_configs)
unspecified_model_config.feature_configs = fixed_feature_configs
premade_lib.verify_config(unspecified_model_config)
specified_model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(specified_feature_configs),
lattices=[['numerical_1', 'categorical'],
['numerical_2', 'categorical']],
num_lattices=2,
lattice_rank=2,
separate_calibrators=True,
output_initialization=[-1.0, 1.0])
premade_lib.verify_config(specified_model_config)
def testSetRandomLattices(self):
random_model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(unspecified_feature_configs),
lattices='random',
num_lattices=3,
lattice_rank=2,
separate_calibrators=True,
output_initialization=[-1.0, 1.0])
premade_lib.set_random_lattice_ensemble(random_model_config)
self.assertLen(random_model_config.lattices, 3)
self.assertListEqual(
[2, 2, 2], [len(lattice) for lattice in random_model_config.lattices])
specified_model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(specified_feature_configs),
lattices=[['numerical_1', 'categorical'],
['numerical_2', 'categorical']],
num_lattices=2,
lattice_rank=2,
separate_calibrators=True,
output_initialization=[-1.0, 1.0])
with self.assertRaisesRegex(
ValueError, 'model_config.lattices must be set to \'random\'.'):
premade_lib.set_random_lattice_ensemble(specified_model_config)
def testLatticeEnsembleH5FormatSaveLoad(self):
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(feature_configs),
lattices=[['numerical_1', 'categorical'],
['numerical_2', 'categorical']],
num_lattices=2,
lattice_rank=2,
separate_calibrators=True,
regularizer_configs=[
configs.RegularizerConfig('calib_hessian', l2=1e-3),
configs.RegularizerConfig('torsion', l2=1e-4),
],
output_min=-1.0,
output_max=1.0,
output_calibration=True,
output_calibration_num_keypoints=5,
output_initialization=[-1.0, 1.0])
model = premade.CalibratedLatticeEnsemble(model_config)
# Compile and fit model.
model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(0.1))
model.fit(fake_data['train_xs'], fake_data['train_ys'])
# Save model using H5 format.
with tempfile.NamedTemporaryFile(suffix='.h5') as f:
tf.keras.models.save_model(model, f.name)
loaded_model = tf.keras.models.load_model(
f.name, custom_objects=premade.get_custom_objects())
self.assertAllClose(model.predict(fake_data['eval_xs']),
loaded_model.predict(fake_data['eval_xs']))
def testLatticeEnsembleFromConfig(self):
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(feature_configs),
lattices=[['numerical_1', 'categorical'],
['numerical_2', 'categorical']],
num_lattices=2,
lattice_rank=2,
separate_calibrators=True,
regularizer_configs=[
configs.RegularizerConfig('calib_hessian', l2=1e-3),
configs.RegularizerConfig('torsion', l2=1e-4),
],
output_min=-1.0,
output_max=1.0,
output_calibration=True,
output_calibration_num_keypoints=5,
output_initialization=[-1.0, 1.0])
model = premade.CalibratedLatticeEnsemble(model_config)
loaded_model = premade.CalibratedLatticeEnsemble.from_config(
model.get_config(), custom_objects=premade.get_custom_objects())
self.assertEqual(
json.dumps(model.get_config(), sort_keys=True, cls=self.Encoder),
json.dumps(loaded_model.get_config(),
sort_keys=True,
cls=self.Encoder))
def testCalibratedLatticeEnsembleModelInfo(self, lattices, num_lattices,
lattice_rank, parameterization,
separate_calibrators,
output_calibration):
self._ResetAllBackends()
feature_configs = copy.deepcopy(self.heart_feature_configs)
if lattices == 'rtl_layer' or parameterization == 'kronecker_factored':
# RTL Layer only supports monotonicity and bound constraints.
for feature_config in feature_configs:
feature_config.lattice_size = 2
feature_config.unimodality = 'none'
feature_config.reflects_trust_in = None
feature_config.dominates = None
feature_config.regularizer_configs = None
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=feature_configs,
lattices=lattices,
num_lattices=num_lattices,
lattice_rank=lattice_rank,
parameterization=parameterization,
separate_calibrators=separate_calibrators,
output_calibration=output_calibration,
)
estimator = estimators.CannedClassifier(
feature_columns=self.heart_feature_columns,
model_config=model_config,
feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1),
prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5),
optimizer=tf.keras.optimizers.Adam(0.01),
prefitting_optimizer=tf.keras.optimizers.Adam(0.01))
estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20))
# Serving input fn is used to create saved models.
serving_input_fn = (
tf.estimator.export.build_parsing_serving_input_receiver_fn(
feature_spec=fc.make_parse_example_spec(
self.heart_feature_columns)))
saved_model_path = estimator.export_saved_model(
estimator.model_dir, serving_input_fn)
logging.info('Model exported to %s', saved_model_path)
model = estimators.get_model_graph(saved_model_path)
expected_num_nodes = (
len(self.heart_feature_columns) + # Input features
num_lattices + # One lattice per submodel
1 + # Averaging submodels
int(output_calibration)) # Output calibration
if separate_calibrators:
expected_num_nodes += num_lattices * lattice_rank
else:
expected_num_nodes += len(self.heart_feature_columns)
self.assertLen(model.nodes, expected_num_nodes)
def testCalibratedLatticeEnsembleCrystals(self):
# Construct model.
self._ResetAllBackends()
model_config = configs.CalibratedLatticeEnsembleConfig(
regularizer_configs=[
configs.RegularizerConfig(name='torsion', l2=1e-4),
configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4),
],
feature_configs=self.heart_feature_configs,
lattices='crystals',
num_lattices=6,
lattice_rank=5,
separate_calibrators=True,
output_calibration=False,
output_min=self.heart_min_label,
output_max=self.heart_max_label - self.numerical_error_epsilon,
output_initialization=[self.heart_min_label, self.heart_max_label],
)
# Perform prefitting steps.
prefitting_model_config = premade_lib.construct_prefitting_model_config(
model_config)
prefitting_model = premade.CalibratedLatticeEnsemble(
prefitting_model_config)
prefitting_model.compile(
loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(0.01))
prefitting_model.fit(
self.heart_train_x,
self.heart_train_y,
batch_size=100,
epochs=50,
verbose=False)
premade_lib.set_crystals_lattice_ensemble(model_config,
prefitting_model_config,
prefitting_model)
# Construct and train final model
model = premade.CalibratedLatticeEnsemble(model_config)
model.compile(
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=tf.keras.metrics.AUC(),
optimizer=tf.keras.optimizers.Adam(0.01))
model.fit(
self.heart_train_x,
self.heart_train_y,
batch_size=100,
epochs=200,
verbose=False)
results = model.evaluate(
self.heart_test_x, self.heart_test_y, verbose=False)
logging.info('Calibrated lattice ensemble classifier results:')
logging.info(results)
self.assertGreater(results[1], 0.85)
def testCalibratedLatticeEnsembleRegressor(self, feature_names, lattices,
num_lattices, lattice_rank,
separate_calibrators,
output_calibration,
average_loss):
self._ResetAllBackends()
feature_columns = [
feature_column for feature_column in self.boston_feature_columns
if feature_column.name in feature_names
]
feature_configs = [
feature_config for feature_config in self.boston_feature_configs
if feature_config.name in feature_names
]
if lattices == 'rtl_layer':
# RTL Layer only supports monotonicity and bound constraints.
feature_configs = copy.deepcopy(feature_configs)
for feature_config in feature_configs:
feature_config.lattice_size = 2
feature_config.unimodality = 'none'
feature_config.reflects_trust_in = None
feature_config.dominates = None
feature_config.regularizer_configs = None
model_config = configs.CalibratedLatticeEnsembleConfig(
regularizer_configs=[
configs.RegularizerConfig(name='torsion', l2=1e-5),
configs.RegularizerConfig(name='output_calib_hessian',
l2=1e-5),
],
feature_configs=feature_configs,
lattices=lattices,
num_lattices=num_lattices,
lattice_rank=lattice_rank,
separate_calibrators=separate_calibrators,
output_calibration=output_calibration,
)
estimator = estimators.CannedRegressor(
feature_columns=feature_columns,
model_config=model_config,
feature_analysis_input_fn=self._GetBostonTrainInputFn(
num_epochs=1),
prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50),
optimizer=tf.keras.optimizers.Adam(0.05),
prefitting_optimizer=tf.keras.optimizers.Adam(0.05))
estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200))
results = estimator.evaluate(input_fn=self._GetBostonTestInputFn())
logging.info('Calibrated lattice ensemble regressor results:')
logging.info(results)
self.assertLess(results['average_loss'], average_loss)
def testCalibratedLatticeEnsembleRTL(self, interpolation, parameterization,
num_terms, expected_minimum_auc):
# Construct model.
self._ResetAllBackends()
rtl_feature_configs = copy.deepcopy(self.heart_feature_configs)
for feature_config in rtl_feature_configs:
feature_config.lattice_size = 2
feature_config.unimodality = 'none'
feature_config.reflects_trust_in = None
feature_config.dominates = None
feature_config.regularizer_configs = None
model_config = configs.CalibratedLatticeEnsembleConfig(
regularizer_configs=[
configs.RegularizerConfig(name='torsion', l2=1e-4),
configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4),
],
feature_configs=rtl_feature_configs,
lattices='rtl_layer',
num_lattices=6,
lattice_rank=5,
interpolation=interpolation,
parameterization=parameterization,
num_terms=num_terms,
separate_calibrators=True,
output_calibration=False,
output_min=self.heart_min_label,
output_max=self.heart_max_label - self.numerical_error_epsilon,
output_initialization=[self.heart_min_label, self.heart_max_label],
)
# We must remove all regularization if using 'kronecker_factored'.
if parameterization == 'kronecker_factored':
model_config.regularizer_configs = None
# Construct and train final model
model = premade.CalibratedLatticeEnsemble(model_config)
model.compile(
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=tf.keras.metrics.AUC(),
optimizer=tf.keras.optimizers.Adam(0.01))
model.fit(
self.heart_train_x,
self.heart_train_y,
batch_size=100,
epochs=200,
verbose=False)
results = model.evaluate(
self.heart_test_x, self.heart_test_y, verbose=False)
logging.info('Calibrated lattice ensemble rtl classifier results:')
logging.info(results)
self.assertGreater(results[1], expected_minimum_auc)
def testCalibratedLatticeEnsembleFix2dConstraintViolations(
self, feature_names, lattices, num_lattices, lattice_rank,
expected_lattices):
self._ResetAllBackends()
feature_columns = [
feature_column for feature_column in self.boston_feature_columns
if feature_column.name in feature_names
]
feature_configs = [
feature_config for feature_config in self.boston_feature_configs
if feature_config.name in feature_names
]
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=feature_configs,
lattices=lattices,
num_lattices=num_lattices,
lattice_rank=lattice_rank,
)
estimator = estimators.CannedRegressor(
feature_columns=feature_columns,
model_config=model_config,
feature_analysis_input_fn=self._GetBostonTrainInputFn(
num_epochs=1),
prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50),
optimizer=tf.keras.optimizers.Adam(0.05),
prefitting_optimizer=tf.keras.optimizers.Adam(0.05))
estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200))
# Serving input fn is used to create saved models.
serving_input_fn = (
tf.estimator.export.build_parsing_serving_input_receiver_fn(
feature_spec=fc.make_parse_example_spec(feature_columns)))
saved_model_path = estimator.export_saved_model(
estimator.model_dir, serving_input_fn)
logging.info('Model exported to %s', saved_model_path)
model = estimators.get_model_graph(saved_model_path)
lattices = []
for node in model.nodes:
if isinstance(node, model_info.LatticeNode):
lattices.append([
input_node.input_node.name
for input_node in node.input_nodes
])
self.assertLen(lattices, len(expected_lattices))
for lattice, expected_lattice in zip(lattices, expected_lattices):
self.assertCountEqual(lattice, expected_lattice)
def testCalibratedLatticeEnsembleModelInfo(self, num_lattices,
lattice_rank,
separate_calibrators,
output_calibration):
self._ResetAllBackends()
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=self.heart_feature_configs,
num_lattices=num_lattices,
lattice_rank=lattice_rank,
separate_calibrators=separate_calibrators,
output_calibration=output_calibration,
)
estimator = estimators.CannedClassifier(
feature_columns=self.heart_feature_columns,
model_config=model_config,
feature_analysis_input_fn=self._GetHeartTrainInputFn(num_epochs=1),
prefitting_input_fn=self._GetHeartTrainInputFn(num_epochs=5),
optimizer=tf.keras.optimizers.Adam(0.01),
prefitting_optimizer=tf.keras.optimizers.Adam(0.01))
estimator.train(input_fn=self._GetHeartTrainInputFn(num_epochs=20))
# Serving input fn is used to create saved models.
serving_input_fn = (
tf.estimator.export.build_parsing_serving_input_receiver_fn(
feature_spec=fc.make_parse_example_spec(
self.heart_feature_columns)))
saved_model_path = estimator.export_saved_model(
estimator.model_dir, serving_input_fn)
logging.info('Model exported to %s', saved_model_path)
model = estimators.get_model_graph(saved_model_path)
expected_num_nodes = (
len(self.heart_feature_columns) + # Input features
num_lattices + # One lattice per submodel
1 + # Averaging submodels
int(output_calibration)) # Output calibration
if separate_calibrators:
expected_num_nodes += num_lattices * lattice_rank
else:
expected_num_nodes += len(self.heart_feature_columns)
self.assertLen(model.nodes, expected_num_nodes)
def testCalibratedLatticeEnsembleRegressor(self, feature_names, lattices,
num_lattices, lattice_rank,
separate_calibrators,
output_calibration,
average_loss):
self._ResetAllBackends()
feature_columns = [
feature_column for feature_column in self.boston_feature_columns
if feature_column.name in feature_names
]
feature_configs = [
feature_config for feature_config in self.boston_feature_configs
if feature_config.name in feature_names
]
model_config = configs.CalibratedLatticeEnsembleConfig(
regularizer_configs=[
configs.RegularizerConfig(name='torsion', l2=1e-5),
configs.RegularizerConfig(name='output_calib_hessian',
l2=1e-5),
],
feature_configs=feature_configs,
lattices=lattices,
num_lattices=num_lattices,
lattice_rank=lattice_rank,
separate_calibrators=separate_calibrators,
output_calibration=output_calibration,
)
estimator = estimators.CannedRegressor(
feature_columns=feature_columns,
model_config=model_config,
feature_analysis_input_fn=self._GetBostonTrainInputFn(
num_epochs=1),
prefitting_input_fn=self._GetBostonTrainInputFn(num_epochs=50),
optimizer=tf.keras.optimizers.Adam(0.05),
prefitting_optimizer=tf.keras.optimizers.Adam(0.05))
estimator.train(input_fn=self._GetBostonTrainInputFn(num_epochs=200))
results = estimator.evaluate(input_fn=self._GetBostonTestInputFn())
logging.info('Calibrated lattice ensemble regressor results:')
logging.info(results)
self.assertLess(results['average_loss'], average_loss)
def testLatticeEnsembleRTLH5FormatSaveLoad(self, parameterization, num_terms):
rtl_feature_configs = copy.deepcopy(feature_configs)
for feature_config in rtl_feature_configs:
feature_config.lattice_size = 2
feature_config.unimodality = 'none'
feature_config.reflects_trust_in = None
feature_config.dominates = None
feature_config.regularizer_configs = None
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=copy.deepcopy(rtl_feature_configs),
lattices='rtl_layer',
num_lattices=2,
lattice_rank=2,
parameterization=parameterization,
num_terms=num_terms,
separate_calibrators=True,
regularizer_configs=[
configs.RegularizerConfig('calib_hessian', l2=1e-3),
configs.RegularizerConfig('torsion', l2=1e-4),
],
output_min=-1.0,
output_max=1.0,
output_calibration=True,
output_calibration_num_keypoints=5,
output_initialization=[-1.0, 1.0])
if parameterization == 'kronecker_factored':
model_config.regularizer_configs = None
model = premade.CalibratedLatticeEnsemble(model_config)
# Compile and fit model.
model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(0.1))
model.fit(fake_data['train_xs'], fake_data['train_ys'])
# Save model using H5 format.
with tempfile.NamedTemporaryFile(suffix='.h5') as f:
tf.keras.models.save_model(model, f.name)
loaded_model = tf.keras.models.load_model(
f.name, custom_objects=premade.get_custom_objects())
self.assertAllClose(
model.predict(fake_data['eval_xs']),
loaded_model.predict(fake_data['eval_xs']))
def test_from_config(self):
feature_configs = [
configs.FeatureConfig(
name='feature_a',
pwl_calibration_input_keypoints='quantiles',
pwl_calibration_num_keypoints=8,
monotonicity=1,
pwl_calibration_clip_max=100,
),
configs.FeatureConfig(
name='feature_b',
lattice_size=3,
unimodality='valley',
pwl_calibration_input_keypoints='uniform',
pwl_calibration_num_keypoints=5,
pwl_calibration_clip_min=130,
pwl_calibration_convexity='convex',
regularizer_configs=[
configs.RegularizerConfig(name='calib_hesian', l2=3e-3),
],
),
configs.FeatureConfig(
name='feature_c',
pwl_calibration_input_keypoints=[0.0, 0.5, 1.0],
reflects_trust_in=[
configs.TrustConfig(feature_name='feature_a'),
configs.TrustConfig(feature_name='feature_b', direction=-1),
],
dominates=[
configs.DominanceConfig(
feature_name='feature_d', dominance_type='monotonic'),
],
),
configs.FeatureConfig(
name='feature_d',
num_buckets=3,
vocabulary_list=['a', 'b', 'c'],
default_value=-1,
),
]
# First we test CalibratedLatticeEnsembleConfig
model_config = configs.CalibratedLatticeEnsembleConfig(
feature_configs=feature_configs,
lattices=[['feature_a', 'feature_b'], ['feature_c', 'feature_d']],
separate_calibrators=True,
regularizer_configs=[
configs.RegularizerConfig('torsion', l2=1e-4),
],
output_min=0.0,
output_max=1.0,
output_calibration=True,
output_calibration_num_keypoints=5,
output_initialization=[0.0, 1.0])
model_config_copy = configs.CalibratedLatticeEnsembleConfig.from_config(
model_config.get_config(), tfl_custom_objects)
self.assertDictEqual(model_config.get_config(),
model_config_copy.get_config())
# Next we test CalibratedLatticeConfig
model_config = configs.CalibratedLatticeConfig(
feature_configs=feature_configs,
regularizer_configs=[
configs.RegularizerConfig('torsion', l2=1e-4),
],
output_min=0.0,
output_max=1.0,
output_calibration=True,
output_calibration_num_keypoints=8,
output_initialization='quantiles')
model_config_copy = configs.CalibratedLatticeConfig.from_config(
model_config.get_config(), tfl_custom_objects)
self.assertDictEqual(model_config.get_config(),
model_config_copy.get_config())
# Last we test CalibratedLinearConfig
model_config = configs.CalibratedLinearConfig(
feature_configs=feature_configs,
regularizer_configs=[
configs.RegularizerConfig('calib_hessian', l2=1e-4),
],
use_bias=True,
output_min=0.0,
output_max=None,
output_calibration=True,
output_initialization='uniform')
model_config_copy = configs.CalibratedLinearConfig.from_config(
model_config.get_config(), tfl_custom_objects)
self.assertDictEqual(model_config.get_config(),
model_config_copy.get_config())
