def testCreateTPUEstimatorSpec(self, n_classes):
"""Tests that an Estimator built with a binary head works."""
train_features, train_labels = test_utils.make_input_data(
256, n_classes)
feature_columns = []
for key in train_features:
feature_columns.append(tf.feature_column.numeric_column(key=key))
head = head_lib._binary_logistic_or_multi_class_head(
n_classes=n_classes,
weight_column=None,
label_vocabulary=None,
loss_reduction=tf.compat.v1.losses.Reduction.NONE)
optimizer = DPGradientDescentGaussianOptimizer(learning_rate=0.5,
l2_norm_clip=1.0,
noise_multiplier=0.0,
num_microbatches=2)
model_fn = make_model_fn(head, optimizer, feature_columns)
classifier = tf_estimator.Estimator(model_fn=model_fn)
classifier.train(input_fn=test_utils.make_input_fn(
train_features, train_labels, True),
steps=4)
test_features, test_labels = test_utils.make_input_data(64, n_classes)
classifier.evaluate(input_fn=test_utils.make_input_fn(
test_features, test_labels, False),
steps=4)
predict_features, predict_labels = test_utils.make_input_data(
64, n_classes)
classifier.predict(input_fn=test_utils.make_input_fn(
predict_features, predict_labels, False))
def __init__(self, config, model_params, model_dir):
self.config = config
self.model_params = model_params
self.model = estimator.Estimator(model_fn=self.my_model,
model_dir=model_dir,
params=self.model_params,
config=self.config)
def simple_fixed_prediction_classifier_identity_label(export_path,
eval_export_path):
"""Exports a simple fixed prediction classifier."""
estimator = tf_estimator.Estimator(
model_fn=fixed_prediction_classifier.model_fn)
estimator.train(input_fn=fixed_prediction_classifier.train_input_fn,
steps=1)
serving_input_receiver_fn = (
tf_estimator.export.build_parsing_serving_input_receiver_fn(
feature_spec={
'classes': tf.io.VarLenFeature(dtype=tf.string),
'scores': tf.io.VarLenFeature(dtype=tf.float32)
}))
eval_input_receiver_fn = export.build_parsing_eval_input_receiver_fn(
feature_spec={
'classes': tf.io.VarLenFeature(dtype=tf.string),
'scores': tf.io.VarLenFeature(dtype=tf.float32),
'label': tf.io.FixedLenFeature([1], dtype=tf.int64),
'language': tf.io.FixedLenFeature([1], dtype=tf.string),
'age': tf.io.FixedLenFeature([1], dtype=tf.float32),
},
label_key='label')
return util.export_model_and_eval_model(
estimator=estimator,
serving_input_receiver_fn=serving_input_receiver_fn,
eval_input_receiver_fn=eval_input_receiver_fn,
export_path=export_path,
eval_export_path=eval_export_path)
def testNoiseMultiplier(self, cls, l2_norm_clip, noise_multiplier,
num_microbatches):
"""Tests that DP optimizers work with tf.estimator."""
linear_regressor = tf_estimator.Estimator(
model_fn=self._make_linear_model_fn(cls,
l2_norm_clip,
noise_multiplier,
num_microbatches,
learning_rate=1.0))
true_weights = np.zeros((1000, 1), dtype=np.float32)
true_bias = np.array([0.0]).astype(np.float32)
train_data = np.zeros((16, 1000), dtype=np.float32)
train_labels = np.matmul(train_data, true_weights) + true_bias
def train_input_fn():
return tf.data.Dataset.from_tensor_slices(
(train_data, train_labels)).batch(16)
linear_regressor.train(input_fn=train_input_fn, steps=1)
kernel_value = linear_regressor.get_variable_value('dense/kernel')
self.assertNear(np.std(kernel_value),
l2_norm_clip * noise_multiplier / num_microbatches,
0.5)
def test_special_tokens_in_estimator(self):
"""Tests getting special tokens without an Eager init context."""
vocab_file = self._make_vocab_file(
["[PAD]", "[UNK]", "[CLS]", "[SEP]", "d", "##ef", "abc", "xy"])
def input_fn():
with tf.init_scope():
self.assertFalse(tf.executing_eagerly())
# Build a preprocessing Model.
sentences = tf.keras.layers.Input(shape=[], dtype=tf.string)
bert_tokenizer = text_layers.BertTokenizer(vocab_file=vocab_file,
lower_case=True)
special_tokens_dict = bert_tokenizer.get_special_tokens_dict()
for k, v in special_tokens_dict.items():
self.assertIsInstance(v, int,
"Unexpected type for {}".format(k))
tokens = bert_tokenizer(sentences)
packed_inputs = text_layers.BertPackInputs(
4, special_tokens_dict=special_tokens_dict)(tokens)
preprocessing = tf.keras.Model(sentences, packed_inputs)
# Map the dataset.
ds = tf.data.Dataset.from_tensors(
(tf.constant(["abc", "DEF"]), tf.constant([0, 1])))
ds = ds.map(lambda features, labels:
(preprocessing(features), labels))
return ds
def model_fn(features, labels, mode):
del labels # Unused.
return tf_estimator.EstimatorSpec(
mode=mode, predictions=features["input_word_ids"])
estimator = tf_estimator.Estimator(model_fn=model_fn)
outputs = list(estimator.predict(input_fn))
self.assertAllEqual(outputs, np.array([[2, 6, 3, 0], [2, 4, 5, 3]]))
def testBaseline(self, cls, num_microbatches):
"""Tests that DP optimizers work with tf.estimator."""
linear_regressor = tf_estimator.Estimator(
model_fn=self._make_linear_model_fn(cls, 100.0, 0.0,
num_microbatches, 0.05))
true_weights = np.array([[-5], [4], [3], [2]]).astype(np.float32)
true_bias = np.array([6.0]).astype(np.float32)
train_data = np.random.normal(scale=3.0,
size=(1000, 4)).astype(np.float32)
train_labels = np.matmul(
train_data, true_weights) + true_bias + np.random.normal(
scale=0.0, size=(1000, 1)).astype(np.float32)
def train_input_fn():
return tf.data.Dataset.from_tensor_slices(
(train_data, train_labels)).batch(8)
linear_regressor.train(input_fn=train_input_fn, steps=125)
self.assertAllClose(
linear_regressor.get_variable_value('dense/kernel'),
true_weights,
atol=0.05)
self.assertAllClose(linear_regressor.get_variable_value('dense/bias'),
true_bias,
atol=0.05)
def load_model(model_dir, inference_batch_size=1024):
"""Loads serialized tf.Estimator model from a dir for inference.
Args:
model_dir: (str) Path to a tf.Estimator model dir.
inference_batch_size: (int) Batch size to use when calling model.predict().
Returns:
(tf.Estimator, dict) A tuple of (model, hparams).
"""
hparams = load_hparams(model_dir)
hparams['batch_size'] = inference_batch_size
model_fn = None
if hparams['model'] == 'rnn':
model_fn = rnn.rnn_model_fn
elif hparams['model'] == 'cnn':
cnn_refs = {} # These are just for debugging and introspection
model_fn = functools.partial(cnn.cnn_model_fn, refs=cnn_refs)
elif hparams['model'] == 'logistic':
model_fn = lr.logistic_regression_model_fn
else:
raise ValueError('Model type "%s" is not supported' % hparams['model'])
model = tf_estimator.Estimator(
model_fn=model_fn,
params=hparams,
model_dir=model_dir,
)
return model, hparams
def main(unused_argv):
logger = tf.get_logger()
logger.set_level(logging.INFO)
# Load training and test data.
train_data, train_labels, test_data, test_labels = load_mnist()
# Instantiate the tf.Estimator.
mnist_classifier = tf_estimator.Estimator(model_fn=cnn_model_fn)
# Create tf.Estimator input functions for the training and test data.
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data},
y=train_labels,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.epochs,
shuffle=True)
eval_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': test_data}, y=test_labels, num_epochs=1, shuffle=False)
# Training loop.
steps_per_epoch = 60000 // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
# Train the model for one epoch.
mnist_classifier.train(input_fn=train_input_fn, steps=steps_per_epoch)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
def testCreateTPUEstimatorSpec(self):
"""Tests that an Estimator built with this head works."""
train_features, train_labels = test_utils.make_multilabel_input_data(
256)
feature_columns = []
for key in train_features:
feature_columns.append(tf.feature_column.numeric_column(key=key))
head = multi_label_head.DPMultiLabelHead(3)
optimizer = DPKerasSGDOptimizer(learning_rate=0.5,
l2_norm_clip=1.0,
noise_multiplier=0.0,
num_microbatches=2)
model_fn = test_utils.make_model_fn(head, optimizer, feature_columns)
classifier = tf_estimator.Estimator(model_fn=model_fn)
classifier.train(input_fn=test_utils.make_input_fn(
train_features, train_labels, True),
steps=4)
test_features, test_labels = test_utils.make_multilabel_input_data(64)
classifier.evaluate(input_fn=test_utils.make_input_fn(
test_features, test_labels, False),
steps=4)
predict_features, predict_labels = test_utils.make_multilabel_input_data(
64)
classifier.predict(input_fn=test_utils.make_input_fn(
predict_features, predict_labels, False))
def simple_batch_size_limited_classifier(export_path, eval_export_path):
"""Exports a simple fixed prediction classifier."""
estimator = tf_estimator.Estimator(model_fn=model_fn)
estimator.train(input_fn=train_input_fn, steps=1)
serving_input_receiver_fn = (
tf_estimator.export.build_parsing_serving_input_receiver_fn(
feature_spec={
'classes': tf.io.FixedLenFeature([], dtype=tf.string),
'scores': tf.io.FixedLenFeature([], dtype=tf.float32)
}))
eval_input_receiver_fn = export.build_parsing_eval_input_receiver_fn(
feature_spec={
'classes': tf.io.FixedLenFeature([], dtype=tf.string),
'scores': tf.io.FixedLenFeature([], dtype=tf.float32),
'labels': tf.io.FixedLenFeature([], dtype=tf.string),
},
label_key='labels')
return util.export_model_and_eval_model(
estimator=estimator,
serving_input_receiver_fn=serving_input_receiver_fn,
eval_input_receiver_fn=eval_input_receiver_fn,
export_path=export_path,
eval_export_path=eval_export_path)
def make_estimator(self):
"""Returns the built `tf.estimator.Estimator` for the TF-Ranking model."""
config = tf_estimator.RunConfig(
model_dir=self._hparams.get("model_dir"),
keep_checkpoint_max=self._hparams.get("num_checkpoints"),
save_checkpoints_secs=self._hparams.get("checkpoint_secs"))
return tf_estimator.Estimator(model_fn=self._model_fn(), config=config)
def main(unused_argv):
logger = tf.get_logger()
logger.set_level(logging.INFO)
if FLAGS.data_l2_norm <= 0:
raise ValueError('data_l2_norm must be positive.')
if FLAGS.dpsgd and FLAGS.learning_rate > 8 / FLAGS.data_l2_norm**2:
raise ValueError(
'The amplification-by-iteration analysis requires'
'learning_rate <= 2 / beta, where beta is the smoothness'
'of the loss function and is upper bounded by ||x||^2 / 4'
'with ||x|| being the largest L2 norm of the samples.')
# Load training and test data.
# Smoothness = ||x||^2 / 4 where ||x|| is the largest L2 norm of the samples.
# To get bounded smoothness, we normalize the data such that each sample has a
# bounded L2 norm.
train_data, train_labels, test_data, test_labels = load_mnist(
data_l2_norm=FLAGS.data_l2_norm)
# Instantiate tf.Estimator.
# pylint: disable=g-long-lambda
model_fn = lambda features, labels, mode: lr_model_fn(
features, labels, mode, nclasses=10, dim=train_data.shape[1:])
mnist_classifier = tf_estimator.Estimator(model_fn=model_fn,
model_dir=FLAGS.model_dir)
# Create tf.Estimator input functions for the training and test data.
# To analyze the per-user privacy loss, we keep the same orders of samples in
# each epoch by setting shuffle=False.
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data},
y=train_labels,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.epochs,
shuffle=False)
eval_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': test_data}, y=test_labels, num_epochs=1, shuffle=False)
# Train the model.
num_samples = train_data.shape[0]
steps_per_epoch = num_samples // FLAGS.batch_size
mnist_classifier.train(input_fn=train_input_fn,
steps=steps_per_epoch * FLAGS.epochs)
# Evaluate the model and print results.
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
print('Test accuracy after {} epochs is: {:.2f}'.format(
FLAGS.epochs, eval_results['accuracy']))
if FLAGS.dpsgd:
print_privacy_guarantees(
epochs=FLAGS.epochs,
batch_size=FLAGS.batch_size,
samples=num_samples,
noise_multiplier=FLAGS.noise_multiplier,
)
def generate_model(model_dir, image_size):
"""Generates a new tensorflow model using estimators"""
return te.Estimator(model_fn=cnn_model_fn,
model_dir=model_dir,
params={
"height": image_size[0],
"width": image_size[1]
})
def main(unused_argv):
logger = tf.get_logger()
logger.set_level(logging.INFO)
if FLAGS.batch_size % FLAGS.microbatches != 0:
raise ValueError(
'Number of microbatches should divide evenly batch_size')
# Load training and test data.
train_data, test_data = load_data()
# Instantiate the tf.Estimator.
conf = tf_estimator.RunConfig(save_summary_steps=1000)
lm_classifier = tf_estimator.Estimator(model_fn=rnn_model_fn,
model_dir=FLAGS.model_dir,
config=conf)
# Create tf.Estimator input functions for the training and test data.
batch_len = FLAGS.batch_size * SEQ_LEN
train_data_end = len(train_data) - len(train_data) % batch_len
test_data_end = len(test_data) - len(test_data) % batch_len
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data[:train_data_end]},
batch_size=batch_len,
num_epochs=FLAGS.epochs,
shuffle=False)
eval_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': test_data[:test_data_end]},
batch_size=batch_len,
num_epochs=1,
shuffle=False)
# Training loop.
steps_per_epoch = len(train_data) // batch_len
for epoch in range(1, FLAGS.epochs + 1):
print('epoch', epoch)
# Train the model for one epoch.
lm_classifier.train(input_fn=train_input_fn, steps=steps_per_epoch)
if epoch % 5 == 0:
name_input_fn = [('Train', train_input_fn),
('Eval', eval_input_fn)]
for name, input_fn in name_input_fn:
# Evaluate the model and print results
eval_results = lm_classifier.evaluate(input_fn=input_fn)
result_tuple = (epoch, eval_results['accuracy'],
eval_results['loss'])
print(
name,
'accuracy after %d epochs is: %.3f (%.4f)' % result_tuple)
# Compute the privacy budget expended so far.
if FLAGS.dpsgd:
eps = compute_epsilon(epoch * steps_per_epoch)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with vanilla non-private SGD optimizer')
def main(unused_argv):
tf.compat.v1.logging.set_verbosity(3)
# Load training and test data.
train_data, train_labels, test_data, test_labels = load_imdb()
# Instantiate the tf.Estimator.
imdb_classifier = tf_estimator.Estimator(model_fn=nn_model_fn,
model_dir=FLAGS.model_dir)
# Create tf.Estimator input functions for the training and test data.
eval_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': test_data}, y=test_labels, num_epochs=1, shuffle=False)
# Training loop.
steps_per_epoch = num_examples // sampling_batch
test_accuracy_list = []
for epoch in range(1, FLAGS.epochs + 1):
for _ in range(steps_per_epoch):
whether = np.random.random_sample(num_examples) > (
1 - sampling_batch / num_examples)
subsampling = [i for i in np.arange(num_examples) if whether[i]]
global microbatches
microbatches = len(subsampling)
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data[subsampling]},
y=train_labels[subsampling],
batch_size=len(subsampling),
num_epochs=1,
shuffle=False)
# Train the model for one step.
imdb_classifier.train(input_fn=train_input_fn, steps=1)
# Evaluate the model and print results
eval_results = imdb_classifier.evaluate(input_fn=eval_input_fn)
test_accuracy = eval_results['accuracy']
test_accuracy_list.append(test_accuracy)
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended so far.
if FLAGS.dpsgd:
eps = compute_eps_poisson(epoch, FLAGS.noise_multiplier,
num_examples, sampling_batch, 1e-5)
mu = compute_mu_poisson(epoch, FLAGS.noise_multiplier,
num_examples, sampling_batch)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
print('For delta=1e-5, the current mu is: %.2f' % mu)
if mu > FLAGS.max_mu:
break
else:
print('Trained with vanilla non-private SGD optimizer')
def legacy_fake_multi_examples_per_input_estimator(export_path,
eval_export_path):
"""Trains and exports a model that treats 1 input as 0 to n examples ."""
estimator = tf_estimator.Estimator(model_fn=_model_fn)
estimator.train(input_fn=_train_input_fn, steps=1)
return util.export_model_and_eval_model(
estimator=estimator,
serving_input_receiver_fn=_serving_input_receiver_fn,
eval_input_receiver_fn=_legacy_eval_input_receiver_fn,
export_path=export_path,
eval_export_path=eval_export_path)
def bad_multi_examples_per_input_estimator_out_of_range_input_refs(
export_path, eval_export_path):
"""Like the above (good) estimator, but the input_refs is out of range."""
estimator = tf_estimator.Estimator(model_fn=_model_fn)
estimator.train(input_fn=_train_input_fn, steps=1)
return util.export_model_and_eval_model(
estimator=estimator,
serving_input_receiver_fn=_serving_input_receiver_fn,
eval_input_receiver_fn=(
_bad_eval_input_receiver_fn_out_of_range_input_refs),
export_path=export_path,
eval_export_path=eval_export_path)
def train_and_eval():
"""Train and Evaluate."""
train_input_fn = make_input_fn(FLAGS.train_path, FLAGS.batch_size)
eval_input_fn = make_input_fn(FLAGS.eval_path,
FLAGS.batch_size,
randomize_input=False,
num_epochs=1)
optimizer = tf.compat.v1.train.AdagradOptimizer(
learning_rate=FLAGS.learning_rate)
def _train_op_fn(loss):
"""Defines train op used in ranking head."""
update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
minimize_op = optimizer.minimize(
loss=loss, global_step=tf.compat.v1.train.get_global_step())
train_op = tf.group([minimize_op, update_ops])
return train_op
ranking_head = tfr.head.create_ranking_head(
loss_fn=tfr.losses.make_loss_fn(
FLAGS.loss, weights_feature_name=FLAGS.weights_feature_name),
eval_metric_fns=eval_metric_fns(),
train_op_fn=_train_op_fn)
estimator = tf_estimator.Estimator(
model_fn=tfr.model.make_groupwise_ranking_fn(
group_score_fn=make_score_fn(),
group_size=FLAGS.group_size,
transform_fn=make_transform_fn(),
ranking_head=ranking_head),
model_dir=FLAGS.model_dir,
config=tf_estimator.RunConfig(save_checkpoints_steps=1000))
train_spec = tf_estimator.TrainSpec(input_fn=train_input_fn,
max_steps=FLAGS.num_train_steps)
exporters = tf_estimator.LatestExporter(
"saved_model_exporter",
serving_input_receiver_fn=make_serving_input_fn())
eval_spec = tf_estimator.EvalSpec(name="eval",
input_fn=eval_input_fn,
steps=1,
exporters=exporters,
start_delay_secs=0,
throttle_secs=15)
# Train and validate.
tf_estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def testClippingNorm(self, cls, num_microbatches):
"""Tests that DP optimizers work with tf.estimator."""
true_weights = np.array([[6.0], [0.0], [0], [0]]).astype(np.float32)
true_bias = np.array([0]).astype(np.float32)
train_data = np.array([[1.0, 0.0, 0.0, 0.0]]).astype(np.float32)
train_labels = np.matmul(train_data, true_weights) + true_bias
def train_input_fn():
return tf.data.Dataset.from_tensor_slices(
(train_data, train_labels)).batch(1)
unclipped_linear_regressor = tf_estimator.Estimator(
model_fn=self._make_linear_model_fn(cls, 1.0e9, 0.0,
num_microbatches, 1.0))
unclipped_linear_regressor.train(input_fn=train_input_fn, steps=1)
kernel_value = unclipped_linear_regressor.get_variable_value(
'dense/kernel')
bias_value = unclipped_linear_regressor.get_variable_value(
'dense/bias')
global_norm = np.linalg.norm(
np.concatenate((kernel_value, [bias_value])))
clipped_linear_regressor = tf_estimator.Estimator(
model_fn=self._make_linear_model_fn(cls, 1.0, 0.0,
num_microbatches, 1.0))
clipped_linear_regressor.train(input_fn=train_input_fn, steps=1)
self.assertAllClose(
clipped_linear_regressor.get_variable_value('dense/kernel'),
kernel_value / global_norm,
atol=0.001)
self.assertAllClose(
clipped_linear_regressor.get_variable_value('dense/bias'),
bias_value / global_norm,
atol=0.001)
def build_estimator(self, model_dir=None, params=None):
"""Produce an Estimator for this Model.
Args:
model_dir: passed in to Estimator - location of tmp files used by
Estimator to checkpoint models
params: passed in to estimator - dict of model_fn parameters
Returns:
a tf.estimator.Estimator
"""
return tf_estimator.Estimator(model_fn=self.build_model_fn(),
model_dir=model_dir,
params=params)
def __init__(self, config):
super().__init__(config)
# ----------------------------------------
self._model_name = 'ResNet50'
self._iter_num = 10
self._train_labels_path = ''
self._validate_labels_path = ''
self._test_labels_path = ''
# ----------------------------------------
self._estimator = estimator.Estimator(
model_fn=self._model_fn,
config=self._config_fn,
params={'fakeParam': 1},
warm_start_from=None
)
def fake_multi_examples_per_input_estimator(export_path,
eval_export_path,
use_iterator=False):
"""Trains and exports a model that treats 1 input as 0 to n examples ."""
estimator = tf_estimator.Estimator(model_fn=_model_fn)
estimator.train(input_fn=_train_input_fn, steps=1)
eval_input_receiver_fn = _eval_input_receiver_fn
if use_iterator:
eval_input_receiver_fn = _eval_input_receiver_using_iterator_fn
return util.export_model_and_eval_model(
estimator=estimator,
serving_input_receiver_fn=_serving_input_receiver_fn,
eval_input_receiver_fn=eval_input_receiver_fn,
export_path=export_path,
eval_export_path=eval_export_path)
def setUp(self):
super(GroupwiseRankingEstimatorTest, self).setUp()
tf.compat.v1.reset_default_graph()
self._model_dir = tf.compat.v1.test.get_temp_dir()
tf.io.gfile.makedirs(self._model_dir)
model_fn = model.make_groupwise_ranking_fn(
_group_score_fn,
group_size=2,
transform_fn=feature.make_identity_transform_fn(
['context', 'weight']),
ranking_head=head.create_ranking_head(
loss_fn=losses.make_loss_fn(
losses.RankingLossKey.PAIRWISE_HINGE_LOSS,
weights_feature_name='weight'),
optimizer=tf.compat.v1.train.AdagradOptimizer(
learning_rate=0.1)))
self._estimator = tf_estimator.Estimator(model_fn, self._model_dir)
def main(unused_argv):
logger = tf.get_logger()
logger.set_level(logging.INFO)
if FLAGS.dpsgd and FLAGS.batch_size % FLAGS.microbatches != 0:
raise ValueError(
'Number of microbatches should divide evenly batch_size')
# Load training and test data.
train_data, train_labels, test_data, test_labels = load_mnist()
# Instantiate the tf.Estimator.
mnist_classifier = tf_estimator.Estimator(model_fn=cnn_model_fn,
model_dir=FLAGS.model_dir)
# Create tf.Estimator input functions for the training and test data.
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data},
y=train_labels,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.epochs,
shuffle=True)
eval_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': test_data}, y=test_labels, num_epochs=1, shuffle=False)
# Training loop.
steps_per_epoch = NUM_TRAIN_EXAMPLES // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
# Train the model for one epoch.
mnist_classifier.train(input_fn=train_input_fn, steps=steps_per_epoch)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended.
if FLAGS.dpsgd:
eps = compute_epsilon(epoch * NUM_TRAIN_EXAMPLES //
FLAGS.batch_size)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with vanilla non-private SGD optimizer')
def train_model(model_fn, train_input_fn, validation_input_fn, params):
"""Trains a model.
Args:
model_fn: (fn) A tf.Estimator model_fn.
train_input_fn: (fn) A tf.Estimator input_fn for the training data.
validation_input_fn: (fn) A tf.Estimator input_fn for the validation data.
params: (dict) Model hyperparameters.
"""
run_config = tf_estimator.RunConfig(
model_dir=FLAGS.model_dir,
save_checkpoints_steps=FLAGS.train_steps_per_eval,
keep_checkpoint_max=None)
logging.warn('RUN CONFIG: %r', run_config)
model = tf_estimator.Estimator(model_fn=model_fn,
params=params,
config=run_config)
experiment = tf.contrib.learn.Experiment(
model,
train_input_fn=train_input_fn,
eval_input_fn=validation_input_fn,
train_steps=FLAGS.max_train_steps,
eval_steps=None,
eval_delay_secs=FLAGS.eval_throttle_secs,
train_steps_per_iteration=FLAGS.train_steps_per_eval)
# WARNING: train_steps_per_iteration should be >= train epoch size, because
# the train input queue is reset upon each evaluation in the Experiment
# implementation currently; i.e., you might only ever train on a subset of the
# training data if you configure train_steps_per_iteration < epoch size.
#
# See https://github.com/tensorflow/tensorflow/issues/11013
precision_early_stopper = train_utils.EarlyStopper(
num_evals_to_wait=FLAGS.early_stopper_num_evals_to_wait,
metric_key=FLAGS.eval_metric)
experiment.continuous_train_and_eval(continuous_eval_predicate_fn=(
precision_early_stopper.early_stop_predicate_fn))
def main(unused_argv):
logging.set_verbosity(logging.INFO)
if FLAGS.dpsgd and FLAGS.batch_size % FLAGS.microbatches != 0:
raise ValueError(
'Number of microbatches should divide evenly batch_size')
# Instantiate the tf.Estimator.
mnist_classifier = tf_estimator.Estimator(model_fn=cnn_model_fn,
model_dir=FLAGS.model_dir)
# Training loop.
steps_per_epoch = 60000 // FLAGS.batch_size
for epoch in range(1, FLAGS.epochs + 1):
start_time = time.time()
# Train the model for one epoch.
mnist_classifier.train(input_fn=common.make_input_fn(
'train', FLAGS.batch_size),
steps=steps_per_epoch)
end_time = time.time()
logging.info('Epoch %d time in seconds: %.2f', epoch,
end_time - start_time)
# Evaluate the model and print results
eval_results = mnist_classifier.evaluate(
input_fn=common.make_input_fn('test', FLAGS.batch_size, 1))
test_accuracy = eval_results['accuracy']
print('Test accuracy after %d epochs is: %.3f' %
(epoch, test_accuracy))
# Compute the privacy budget expended.
if FLAGS.dpsgd:
if FLAGS.noise_multiplier > 0.0:
eps, _ = compute_dp_sgd_privacy_lib.compute_dp_sgd_privacy(
60000, FLAGS.batch_size, FLAGS.noise_multiplier, epoch,
1e-5)
print('For delta=1e-5, the current epsilon is: %.2f' % eps)
else:
print('Trained with DP-SGD but with zero noise.')
else:
print('Trained with vanilla non-private SGD optimizer')
def testEstimator(self):
"""Tests that DP optimizers work with tf.estimator."""
def linear_model_fn(features, labels, mode):
preds = tf.keras.layers.Dense(1, activation='linear',
name='dense')(features['x'])
vector_loss = tf.math.squared_difference(labels, preds)
scalar_loss = tf.reduce_mean(input_tensor=vector_loss)
optimizer = VectorizedDPSGD(l2_norm_clip=1.0,
noise_multiplier=0.,
num_microbatches=1,
learning_rate=1.0)
global_step = tf.compat.v1.train.get_global_step()
train_op = optimizer.minimize(loss=vector_loss,
global_step=global_step)
return tf_estimator.EstimatorSpec(mode=mode,
loss=scalar_loss,
train_op=train_op)
linear_regressor = tf_estimator.Estimator(model_fn=linear_model_fn)
true_weights = np.array([[-5], [4], [3], [2]]).astype(np.float32)
true_bias = 6.0
train_data = np.random.normal(scale=3.0,
size=(200, 4)).astype(np.float32)
train_labels = np.matmul(
train_data, true_weights) + true_bias + np.random.normal(
scale=0.1, size=(200, 1)).astype(np.float32)
train_input_fn = tf_compat_v1_estimator.inputs.numpy_input_fn(
x={'x': train_data},
y=train_labels,
batch_size=20,
num_epochs=10,
shuffle=True)
linear_regressor.train(input_fn=train_input_fn, steps=100)
self.assertAllClose(
linear_regressor.get_variable_value('dense/kernel'),
true_weights,
atol=1.0)
def main(unused_argv):
config = tf_estimator.RunConfig()
classifier = tf_estimator.Estimator(get_model_fn(), config=config)
def _merge_datasets(test_batch):
feature, label = test_batch['image'], test_batch['label'],
features = {
'feature': feature,
}
labels = {
'label': label,
}
return (features, labels)
def get_dataset(dataset_split):
"""Returns dataset creation function."""
def make_input_dataset():
"""Returns input dataset."""
test_data = tfds.load(name=FLAGS.target_dataset, split=dataset_split)
test_data = test_data.batch(FLAGS.train_batch_size)
dataset = tf.data.Dataset.zip((test_data,))
dataset = dataset.map(_merge_datasets)
dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
return dataset
return make_input_dataset
num_eval_images = NUM_EVAL_IMAGES[FLAGS.target_dataset]
eval_steps = num_eval_images // FLAGS.train_batch_size
classifier.evaluate(
input_fn=get_dataset('test'),
steps=eval_steps,
checkpoint_path=FLAGS.ckpt_path,
)
def test4():
(X_train2, y_train2), (X_test2, y_test2) = ld.loadFakeDataPandas(5, 2, 0.1, 15)
print(X_test2)
print(y_test2)
my_feature_columns = []
columnNames = ld.genColumnNames(5)
for key in columnNames:
my_feature_columns.append(tf.feature_column.numeric_column(key=key))
regressor = estimator.Estimator(
model_fn=cE.myCustomEstimator,
params={
"feature_columns": my_feature_columns,
"learning_rate": 0.001,
"optimizer": tf.train.AdamOptimizer,
"hidden_units": [20, 20]
})
regressor.train(input_fn=lambda: training_input_fn_Slices(X_train2, y_train2, 1000), steps=1000)
eval_dict = regressor.evaluate(input_fn=lambda: eval_input_fn(X_test2, y_test2, 1000))
print("eval: " + str(eval_dict))
debug_pred = regressor.predict(input_fn=lambda: eval_input_fn(X_test2, labels=None, batch_size=1000))
# debug_predicted = [p['predictions'] for p in debug_pred]
for i in debug_pred:
print(i)
print("success!")
def get_estimator(run_config, hparams):
return estimator.Estimator(
model_fn=model_fn,
params=hparams,
config=run_config,
)
