def test_tf_optimizer_with_sparse_gradient(self):
import tensorflow as tf
ids = np.random.randint(0, 10, size=[40])
labels = np.random.randint(0, 5, size=[40])
id_rdd = self.sc.parallelize(ids)
label_rdd = self.sc.parallelize(labels)
training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]])
with tf.Graph().as_default():
dataset = TFDataset.from_rdd(training_rdd,
names=["ids", "labels"],
shapes=[[], []],
types=[tf.int32, tf.int32],
batch_size=8)
id_tensor, label_tensor = dataset.tensors
embedding_table = tf.get_variable(name="word_embedding",
shape=[10, 5])
embedding = tf.nn.embedding_lookup(embedding_table, id_tensor)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=embedding,
labels=label_tensor))
optimizer = TFOptimizer(loss, Adam(1e-3))
optimizer.optimize(end_trigger=MaxEpoch(1))
optimizer.sess.close()
def train(self, input_fn, steps=None):
"""Trains a model given training data `input_fn`.
:param input_fn: A function that constructs the input data for evaluation. The
function should construct and return one of the following:
* A `TFDataset` object, each elements of which is a tuple `(features, labels)`.
* A `tf.data.Dataset` object: Outputs of `Dataset` object must be a tuple
`(features, labels)` with same constraints as below.
* A tuple `(features, labels)`: Where `features` is a `tf.Tensor` or a dictionary
of string feature name to `Tensor` and `labels` is a `Tensor` or a
dictionary of string label name to `Tensor`. Both `features` and
`labels` are consumed by `model_fn`. They should satisfy the expectation
of `model_fn` from inputs.
:param steps: Number of steps for which to train the model.
Returns:
`self`, for chaining.
"""
with tf.Graph().as_default() as g:
global_step_tensor = self.estimator._create_and_assert_global_step(
g)
add_step_input = tf.placeholder(dtype=tf.int64, shape=())
assign_step = tf.assign_add(global_step_tensor, add_step_input)
result = self.estimator._call_input_fn(input_fn,
tf.estimator.ModeKeys.TRAIN)
if isinstance(result, TFDataset):
if not result.has_batch:
raise ValueError("The batch_size of TFDataset must be " +
"specified when used for training.")
spec = self._call_model_fn(result.feature_tensors,
result.label_tensors,
tf.estimator.ModeKeys.TRAIN,
self.config)
optim_method = TFOptimizer.to_bigdl_optim_method(
koptim_method=self.optimizer)
latest_checkpoint = self.estimator.latest_checkpoint()
with tf.Session() as sess:
saver = tf.train.Saver()
if latest_checkpoint:
saver.restore(sess, latest_checkpoint)
else:
sess.run(tf.global_variables_initializer())
opt = TFOptimizer.from_loss(
spec.loss,
optim_method,
session=sess,
clip_norm=self.gradient_clipping_norm,
clip_value=self.gradient_clipping_constant)
opt.optimize(MaxIteration(steps))
sess.run(assign_step, feed_dict={add_step_input: steps})
final_step = sess.run(global_step_tensor)
saver.save(sess,
self.estimator.model_dir + "/model",
global_step=final_step)
return self
return self.estimator.train(input_fn, steps=steps)
def test_tf_optimizer_with_sparse_gradient_using_keras(self):
import tensorflow as tf
ids = np.random.randint(0, 10, size=[40])
labels = np.random.randint(0, 5, size=[40])
id_rdd = self.sc.parallelize(ids)
label_rdd = self.sc.parallelize(labels)
training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]])
dataset = TFDataset.from_rdd(training_rdd,
names=["ids", "labels"],
shapes=[[], []],
types=[tf.int32, tf.int32],
batch_size=8)
words_input = tf.keras.layers.Input(shape=(), name='words_input')
embedding_layer = tf.keras.layers.Embedding(input_dim=10,
output_dim=5,
name='word_embedding')
word_embeddings = embedding_layer(words_input)
embedding = tf.keras.layers.Flatten()(word_embeddings)
output = tf.keras.layers.Dense(5, activation="softmax")(embedding)
model = tf.keras.models.Model(inputs=[words_input], outputs=[output])
model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy")
optimizer = TFOptimizer.from_keras(model, dataset)
optimizer.optimize()
def test_tf_optimizer_with_sparse_gradient_using_keras(self):
import tensorflow as tf
ids = np.random.randint(0, 10, size=[40])
labels = np.random.randint(0, 5, size=[40])
id_rdd = self.sc.parallelize(ids)
label_rdd = self.sc.parallelize(labels)
training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]])
dataset = TFDataset.from_rdd(training_rdd,
names=["ids", "labels"],
shapes=[[], []],
types=[tf.int32, tf.int32],
batch_size=8)
from tensorflow.python.ops import variable_scope
def variable_creator(**kwargs):
kwargs["use_resource"] = False
return variable_scope.default_variable_creator(None, **kwargs)
getter = lambda next_creator, **kwargs: variable_creator(**kwargs)
with variable_scope.variable_creator_scope(getter):
words_input = tf.keras.layers.Input(shape=(), name='words_input')
embedding_layer = tf.keras.layers.Embedding(input_dim=10,
output_dim=5,
name='word_embedding')
word_embeddings = embedding_layer(words_input)
embedding = tf.keras.layers.Flatten()(word_embeddings)
output = tf.keras.layers.Dense(5, activation="softmax")(embedding)
model = tf.keras.models.Model(inputs=[words_input],
outputs=[output])
model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy")\
optimizer = TFOptimizer.from_keras(model, dataset)
optimizer.optimize()
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs):
if not self.tf_optimizer:
self.tf_optimizer = TFOptimizer.from_keras(
self.model, dataset, val_spilt=validation_split, **kwargs)
else:
self.tf_optimizer.refresh_weights()
end_epoch = self.tf_optimizer_done_epochs + epochs
self.tf_optimizer.optimize(MaxEpoch(end_epoch))
self.tf_optimizer_done_epochs = end_epoch
def main():
sc = init_nncontext()
# get data, pre-process and create TFDataset
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
image_rdd = sc.parallelize(images_data)
labels_rdd = sc.parallelize(labels_data)
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
dataset = TFDataset.from_rdd(rdd,
names=["features", "labels"],
shapes=[(None, 28, 28, 1), (None, 1)],
types=[tf.float32, tf.int32]
)
# construct the model from TFDataset
images, labels = dataset.inputs
labels = tf.squeeze(labels)
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images, num_classes=10, is_training=True)
loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss, Adam(1e-3))
# kick off training
# you may change the MaxIteration to MaxEpoch(5) to make it converge
optimizer.optimize(end_trigger=MaxIteration(20), batch_size=280)
# evaluate
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test")
images_data = normalizer(images_data, mnist.TRAIN_MEAN, mnist.TRAIN_STD)
predictions = tf.argmax(logits, axis=1)
predictions_data, loss_value = optimizer.sess.run([predictions, loss],
feed_dict={images: images_data,
labels: labels_data})
print(np.mean(np.equal(predictions_data, labels_data)))
print(loss_value)
def train(self, input_fn, steps=None):
with tf.Graph().as_default() as g:
global_step_tensor = self.estimator._create_and_assert_global_step(
g)
add_step_input = tf.placeholder(dtype=tf.int64, shape=())
assign_step = tf.assign_add(global_step_tensor, add_step_input)
result = self.estimator._call_input_fn(input_fn,
tf.estimator.ModeKeys.TRAIN)
if isinstance(result, TFDataset):
if not result.has_batch:
raise ValueError("The batch_size of TFDataset must be " +
"specified when used for training.")
spec = self._call_model_fn(result.feature_tensors,
result.label_tensors,
tf.estimator.ModeKeys.TRAIN,
self.config)
optim_method = TFOptimizer.to_bigdl_optim_method(
koptim_method=self.optimizer)
latest_checkpoint = self.estimator.latest_checkpoint()
with tf.Session() as sess:
saver = tf.train.Saver()
if latest_checkpoint:
saver.restore(sess, latest_checkpoint)
else:
sess.run(tf.global_variables_initializer())
opt = TFOptimizer.from_loss(spec.loss,
optim_method,
session=sess)
opt.optimize(MaxIteration(steps))
sess.run(assign_step, feed_dict={add_step_input: steps})
final_step = sess.run(global_step_tensor)
saver.save(sess,
self.estimator.model_dir + "/model",
global_step=final_step)
return self
return self.estimator.train(input_fn, steps=steps)
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs):
self.tf_optimizer = TFOptimizer.from_keras(self.model,
dataset,
val_split=validation_split,
**kwargs)
if self.train_summary is not None:
self.tf_optimizer.set_train_summary(self.train_summary)
if self.val_summary is not None:
self.tf_optimizer.set_val_summary(self.val_summary)
self.tf_optimizer.optimize(MaxEpoch(epochs))
def main(max_epoch, data_num):
sc = init_nncontext()
# get data, pre-process and create TFDataset
def get_data_rdd(dataset):
(images_data,
labels_data) = mnist.read_data_sets("/tmp/mnist", dataset)
image_rdd = sc.parallelize(images_data[:data_num])
labels_rdd = sc.parallelize(labels_data[:data_num])
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
return rdd
training_rdd = get_data_rdd("train")
testing_rdd = get_data_rdd("test")
dataset = TFDataset.from_rdd(training_rdd,
names=["features", "labels"],
shapes=[[28, 28, 1], []],
types=[tf.float32, tf.int32],
batch_size=280,
val_rdd=testing_rdd)
# construct the model from TFDataset
images, labels = dataset.tensors
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss,
Adam(1e-3),
val_outputs=[logits],
val_labels=[labels],
val_method=Top1Accuracy())
optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet"))
optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet"))
# kick off training
optimizer.optimize(end_trigger=MaxEpoch(max_epoch))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/")
def main(max_epoch, data_num):
sc = init_nncontext()
# get data, pre-process and create TFDataset
def get_data_rdd(dataset):
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset)
image_rdd = sc.parallelize(images_data[:data_num])
labels_rdd = sc.parallelize(labels_data[:data_num])
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
return rdd
training_rdd = get_data_rdd("train")
testing_rdd = get_data_rdd("test")
dataset = TFDataset.from_rdd(training_rdd,
names=["features", "labels"],
shapes=[[28, 28, 1], []],
types=[tf.float32, tf.int32],
batch_size=280,
val_rdd=testing_rdd
)
data = Input(shape=[28, 28, 1])
x = Flatten()(data)
x = Dense(64, activation='relu')(x)
x = Dense(64, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(input=data, output=predictions)
model.compile(optimizer='rmsprop',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
optimizer = TFOptimizer.from_keras(model, dataset)
optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet"))
optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet"))
# kick off training
optimizer.optimize(end_trigger=MaxEpoch(max_epoch))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/")
def test_tfdataset_with_tfrecord(self):
train_path = os.path.join(resource_path,
"tfrecord/mnist_train.tfrecord")
test_path = os.path.join(resource_path, "tfrecord/mnist_test.tfrecord")
dataset = TFDataset.from_tfrecord_file(self.sc,
train_path,
batch_size=8,
validation_file_path=test_path)
dataset = dataset.map(lambda x: parse_fn(x[0]))
flat = tf.layers.flatten(dataset.feature_tensors)
logits = tf.layers.dense(flat, 10)
labels = dataset.label_tensors
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits,
labels=labels))
opt = TFOptimizer.from_loss(loss, Adam())
opt.optimize()
def test_tf_optimizer_metrics(self):
features = np.random.randn(20, 10)
labels = np.random.randint(0, 10, size=[20])
with tf.Graph().as_default():
dataset = TFDataset.from_ndarrays((features, labels),
batch_size=4,
val_tensors=(features, labels))
feature_tensor, label_tensor = dataset.tensors
output = tf.layers.dense(feature_tensor, 10)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=output,
labels=label_tensor))
optimizer = TFOptimizer.from_loss(loss,
Adam(1e-3),
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
metrics={"loss": loss})
optimizer.optimize(end_trigger=MaxEpoch(1))
optimizer.sess.close()
def test_tf_optimizer_metrics(self):
features = np.random.randn(20, 10)
labels = np.random.randint(0, 10, size=[20])
with tf.Graph().as_default():
dataset = TFDataset.from_ndarrays((features, labels),
batch_size=4,
val_tensors=(features, labels))
feature_tensor, label_tensor = dataset.tensors
features = tf.layers.dense(feature_tensor, 8)
output = tf.layers.dense(features, 10)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=output,
labels=label_tensor))
optimizer = TFOptimizer.from_loss(loss, {
"dense/": Adam(1e-3),
"dense_1/": SGD(0.0)
},
val_outputs=[output],
val_labels=[label_tensor],
val_method=Accuracy(),
metrics={"loss": loss})
initial_weights = optimizer.tf_model.training_helper_layer.get_weights(
)
optimizer.optimize(end_trigger=MaxEpoch(1))
updated_weights = optimizer.tf_model.training_helper_layer.get_weights(
)
for i in [
0, 1
]: # weights and bias combined with "dense/" should be updated
assert not np.allclose(initial_weights[i], updated_weights[i])
for i in [
2, 3
]: # weights and bias combined with "dense_1" should be unchanged
assert np.allclose(initial_weights[i], updated_weights[i])
optimizer.sess.close()
def main():
sc = init_nncontext()
# get data, pre-process and create TFDataset
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
image_rdd = sc.parallelize(images_data)
labels_rdd = sc.parallelize(labels_data)
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
dataset = TFDataset.from_rdd(rdd,
names=["features", "labels"],
shapes=[[28, 28, 1], [1]],
types=[tf.float32, tf.int32],
batch_size=280)
# construct the model from TFDataset
images, labels = dataset.tensors
labels = tf.squeeze(labels)
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss, Adam(1e-3))
optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet"))
# kick off training
for i in range(5):
optimizer.optimize(end_trigger=MaxEpoch(i + 1))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/")
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs):
self.tf_optimizer = TFOptimizer.from_keras(self.model,
dataset,
val_spilt=validation_split,
**kwargs)
self.tf_optimizer.optimize(MaxEpoch(epochs))
output_dim=FC_LINEAR_DIMENSION, # 尺寸: 32 -> 64.
activation="sigmoid"))
# BigDL 不支持 parameter sharing, 不得已而为之.
both_feature = TimeDistributed(layer=convolve_net,
input_shape=input_shape)(both_input)
encode_left = both_feature.index_select(1, 0)
encode_right = both_feature.index_select(1, 1)
distance = autograd.abs(encode_left - encode_right)
predict = Dense(output_dim=NUM_CLASS_LABEL, activation="sigmoid")(distance)
siamese_net = Model(input=both_input, output=predict)
siamese_net.compile(optimizer="adam",
loss='sparse_categorical_crossentropy',
metrics=["accuracy"])
# 构造分布式的数据集对象.
data_set = TFDataset.from_rdd(train_rdd,
shapes=[input_shape, [1]],
batch_size=args.batch_size,
val_rdd=test_rdd)
optimizer = TFOptimizer.from_keras(siamese_net, data_set)
app_name = "Siamese Network"
optimizer.set_train_summary(TrainSummary("tmp", app_name))
optimizer.set_val_summary(ValidationSummary("tmp", app_name))
optimizer.optimize(end_trigger=MaxEpoch(args.num_epoch))
