def test_tf_optimizer_with_sparse_gradient(self):
import tensorflow as tf
ids = np.random.randint(0, 10, size=[40])
labels = np.random.randint(0, 5, size=[40])
id_rdd = self.sc.parallelize(ids)
label_rdd = self.sc.parallelize(labels)
training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]])
with tf.Graph().as_default():
dataset = TFDataset.from_rdd(training_rdd,
names=["ids", "labels"],
shapes=[[], []],
types=[tf.int32, tf.int32],
batch_size=8)
id_tensor, label_tensor = dataset.tensors
embedding_table = tf.get_variable(name="word_embedding",
shape=[10, 5])
embedding = tf.nn.embedding_lookup(embedding_table, id_tensor)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=embedding,
labels=label_tensor))
optimizer = TFOptimizer(loss, Adam(1e-3))
optimizer.optimize(end_trigger=MaxEpoch(1))
optimizer.sess.close()
def main(max_epoch, data_num):
sc = init_nncontext()
# get data, pre-process and create TFDataset
def get_data_rdd(dataset):
(images_data,
labels_data) = mnist.read_data_sets("/tmp/mnist", dataset)
image_rdd = sc.parallelize(images_data[:data_num])
labels_rdd = sc.parallelize(labels_data[:data_num])
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
return rdd
training_rdd = get_data_rdd("train")
testing_rdd = get_data_rdd("test")
dataset = TFDataset.from_rdd(training_rdd,
names=["features", "labels"],
shapes=[[28, 28, 1], []],
types=[tf.float32, tf.int32],
batch_size=280,
val_rdd=testing_rdd)
# construct the model from TFDataset
images, labels = dataset.tensors
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss,
Adam(1e-3),
val_outputs=[logits],
val_labels=[labels],
val_method=Top1Accuracy())
optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet"))
optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet"))
# kick off training
optimizer.optimize(end_trigger=MaxEpoch(max_epoch))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/")
def main():
sc = init_nncontext()
# get data, pre-process and create TFDataset
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
image_rdd = sc.parallelize(images_data)
labels_rdd = sc.parallelize(labels_data)
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
dataset = TFDataset.from_rdd(rdd,
names=["features", "labels"],
shapes=[(None, 28, 28, 1), (None, 1)],
types=[tf.float32, tf.int32]
)
# construct the model from TFDataset
images, labels = dataset.inputs
labels = tf.squeeze(labels)
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images, num_classes=10, is_training=True)
loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss, Adam(1e-3))
# kick off training
# you may change the MaxIteration to MaxEpoch(5) to make it converge
optimizer.optimize(end_trigger=MaxIteration(20), batch_size=280)
# evaluate
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test")
images_data = normalizer(images_data, mnist.TRAIN_MEAN, mnist.TRAIN_STD)
predictions = tf.argmax(logits, axis=1)
predictions_data, loss_value = optimizer.sess.run([predictions, loss],
feed_dict={images: images_data,
labels: labels_data})
print(np.mean(np.equal(predictions_data, labels_data)))
print(loss_value)
def main():
sc = init_nncontext()
# get data, pre-process and create TFDataset
(images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train")
image_rdd = sc.parallelize(images_data)
labels_rdd = sc.parallelize(labels_data)
rdd = image_rdd.zip(labels_rdd) \
.map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD),
np.array(rec_tuple[1])])
dataset = TFDataset.from_rdd(rdd,
names=["features", "labels"],
shapes=[[28, 28, 1], [1]],
types=[tf.float32, tf.int32],
batch_size=280)
# construct the model from TFDataset
images, labels = dataset.tensors
labels = tf.squeeze(labels)
with slim.arg_scope(lenet.lenet_arg_scope()):
logits, end_points = lenet.lenet(images,
num_classes=10,
is_training=True)
loss = tf.reduce_mean(
tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
# create a optimizer
optimizer = TFOptimizer(loss, Adam(1e-3))
optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet"))
# kick off training
for i in range(5):
optimizer.optimize(end_trigger=MaxEpoch(i + 1))
saver = tf.train.Saver()
saver.save(optimizer.sess, "/tmp/lenet/")
