def test_tf_optimizer_with_sparse_gradient(self): import tensorflow as tf ids = np.random.randint(0, 10, size=[40]) labels = np.random.randint(0, 5, size=[40]) id_rdd = self.sc.parallelize(ids) label_rdd = self.sc.parallelize(labels) training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]]) with tf.Graph().as_default(): dataset = TFDataset.from_rdd(training_rdd, names=["ids", "labels"], shapes=[[], []], types=[tf.int32, tf.int32], batch_size=8) id_tensor, label_tensor = dataset.tensors embedding_table = tf.get_variable(name="word_embedding", shape=[10, 5]) embedding = tf.nn.embedding_lookup(embedding_table, id_tensor) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=embedding, labels=label_tensor)) optimizer = TFOptimizer(loss, Adam(1e-3)) optimizer.optimize(end_trigger=MaxEpoch(1)) optimizer.sess.close()
def train(self, input_fn, steps=None): """Trains a model given training data `input_fn`. :param input_fn: A function that constructs the input data for evaluation. The function should construct and return one of the following: * A `TFDataset` object, each elements of which is a tuple `(features, labels)`. * A `tf.data.Dataset` object: Outputs of `Dataset` object must be a tuple `(features, labels)` with same constraints as below. * A tuple `(features, labels)`: Where `features` is a `tf.Tensor` or a dictionary of string feature name to `Tensor` and `labels` is a `Tensor` or a dictionary of string label name to `Tensor`. Both `features` and `labels` are consumed by `model_fn`. They should satisfy the expectation of `model_fn` from inputs. :param steps: Number of steps for which to train the model. Returns: `self`, for chaining. """ with tf.Graph().as_default() as g: global_step_tensor = self.estimator._create_and_assert_global_step( g) add_step_input = tf.placeholder(dtype=tf.int64, shape=()) assign_step = tf.assign_add(global_step_tensor, add_step_input) result = self.estimator._call_input_fn(input_fn, tf.estimator.ModeKeys.TRAIN) if isinstance(result, TFDataset): if not result.has_batch: raise ValueError("The batch_size of TFDataset must be " + "specified when used for training.") spec = self._call_model_fn(result.feature_tensors, result.label_tensors, tf.estimator.ModeKeys.TRAIN, self.config) optim_method = TFOptimizer.to_bigdl_optim_method( koptim_method=self.optimizer) latest_checkpoint = self.estimator.latest_checkpoint() with tf.Session() as sess: saver = tf.train.Saver() if latest_checkpoint: saver.restore(sess, latest_checkpoint) else: sess.run(tf.global_variables_initializer()) opt = TFOptimizer.from_loss( spec.loss, optim_method, session=sess, clip_norm=self.gradient_clipping_norm, clip_value=self.gradient_clipping_constant) opt.optimize(MaxIteration(steps)) sess.run(assign_step, feed_dict={add_step_input: steps}) final_step = sess.run(global_step_tensor) saver.save(sess, self.estimator.model_dir + "/model", global_step=final_step) return self return self.estimator.train(input_fn, steps=steps)
def test_tf_optimizer_with_sparse_gradient_using_keras(self): import tensorflow as tf ids = np.random.randint(0, 10, size=[40]) labels = np.random.randint(0, 5, size=[40]) id_rdd = self.sc.parallelize(ids) label_rdd = self.sc.parallelize(labels) training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]]) dataset = TFDataset.from_rdd(training_rdd, names=["ids", "labels"], shapes=[[], []], types=[tf.int32, tf.int32], batch_size=8) words_input = tf.keras.layers.Input(shape=(), name='words_input') embedding_layer = tf.keras.layers.Embedding(input_dim=10, output_dim=5, name='word_embedding') word_embeddings = embedding_layer(words_input) embedding = tf.keras.layers.Flatten()(word_embeddings) output = tf.keras.layers.Dense(5, activation="softmax")(embedding) model = tf.keras.models.Model(inputs=[words_input], outputs=[output]) model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy") optimizer = TFOptimizer.from_keras(model, dataset) optimizer.optimize()
def test_tf_optimizer_with_sparse_gradient_using_keras(self): import tensorflow as tf ids = np.random.randint(0, 10, size=[40]) labels = np.random.randint(0, 5, size=[40]) id_rdd = self.sc.parallelize(ids) label_rdd = self.sc.parallelize(labels) training_rdd = id_rdd.zip(label_rdd).map(lambda x: [x[0], x[1]]) dataset = TFDataset.from_rdd(training_rdd, names=["ids", "labels"], shapes=[[], []], types=[tf.int32, tf.int32], batch_size=8) from tensorflow.python.ops import variable_scope def variable_creator(**kwargs): kwargs["use_resource"] = False return variable_scope.default_variable_creator(None, **kwargs) getter = lambda next_creator, **kwargs: variable_creator(**kwargs) with variable_scope.variable_creator_scope(getter): words_input = tf.keras.layers.Input(shape=(), name='words_input') embedding_layer = tf.keras.layers.Embedding(input_dim=10, output_dim=5, name='word_embedding') word_embeddings = embedding_layer(words_input) embedding = tf.keras.layers.Flatten()(word_embeddings) output = tf.keras.layers.Dense(5, activation="softmax")(embedding) model = tf.keras.models.Model(inputs=[words_input], outputs=[output]) model.compile(optimizer="sgd", loss="sparse_categorical_crossentropy")\ optimizer = TFOptimizer.from_keras(model, dataset) optimizer.optimize()
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs): if not self.tf_optimizer: self.tf_optimizer = TFOptimizer.from_keras( self.model, dataset, val_spilt=validation_split, **kwargs) else: self.tf_optimizer.refresh_weights() end_epoch = self.tf_optimizer_done_epochs + epochs self.tf_optimizer.optimize(MaxEpoch(end_epoch)) self.tf_optimizer_done_epochs = end_epoch
def main(): sc = init_nncontext() # get data, pre-process and create TFDataset (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train") image_rdd = sc.parallelize(images_data) labels_rdd = sc.parallelize(labels_data) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) dataset = TFDataset.from_rdd(rdd, names=["features", "labels"], shapes=[(None, 28, 28, 1), (None, 1)], types=[tf.float32, tf.int32] ) # construct the model from TFDataset images, labels = dataset.inputs labels = tf.squeeze(labels) with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) # create a optimizer optimizer = TFOptimizer(loss, Adam(1e-3)) # kick off training # you may change the MaxIteration to MaxEpoch(5) to make it converge optimizer.optimize(end_trigger=MaxIteration(20), batch_size=280) # evaluate (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test") images_data = normalizer(images_data, mnist.TRAIN_MEAN, mnist.TRAIN_STD) predictions = tf.argmax(logits, axis=1) predictions_data, loss_value = optimizer.sess.run([predictions, loss], feed_dict={images: images_data, labels: labels_data}) print(np.mean(np.equal(predictions_data, labels_data))) print(loss_value)
def train(self, input_fn, steps=None): with tf.Graph().as_default() as g: global_step_tensor = self.estimator._create_and_assert_global_step( g) add_step_input = tf.placeholder(dtype=tf.int64, shape=()) assign_step = tf.assign_add(global_step_tensor, add_step_input) result = self.estimator._call_input_fn(input_fn, tf.estimator.ModeKeys.TRAIN) if isinstance(result, TFDataset): if not result.has_batch: raise ValueError("The batch_size of TFDataset must be " + "specified when used for training.") spec = self._call_model_fn(result.feature_tensors, result.label_tensors, tf.estimator.ModeKeys.TRAIN, self.config) optim_method = TFOptimizer.to_bigdl_optim_method( koptim_method=self.optimizer) latest_checkpoint = self.estimator.latest_checkpoint() with tf.Session() as sess: saver = tf.train.Saver() if latest_checkpoint: saver.restore(sess, latest_checkpoint) else: sess.run(tf.global_variables_initializer()) opt = TFOptimizer.from_loss(spec.loss, optim_method, session=sess) opt.optimize(MaxIteration(steps)) sess.run(assign_step, feed_dict={add_step_input: steps}) final_step = sess.run(global_step_tensor) saver.save(sess, self.estimator.model_dir + "/model", global_step=final_step) return self return self.estimator.train(input_fn, steps=steps)
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs): self.tf_optimizer = TFOptimizer.from_keras(self.model, dataset, val_split=validation_split, **kwargs) if self.train_summary is not None: self.tf_optimizer.set_train_summary(self.train_summary) if self.val_summary is not None: self.tf_optimizer.set_val_summary(self.val_summary) self.tf_optimizer.optimize(MaxEpoch(epochs))
def main(max_epoch, data_num): sc = init_nncontext() # get data, pre-process and create TFDataset def get_data_rdd(dataset): (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset) image_rdd = sc.parallelize(images_data[:data_num]) labels_rdd = sc.parallelize(labels_data[:data_num]) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) return rdd training_rdd = get_data_rdd("train") testing_rdd = get_data_rdd("test") dataset = TFDataset.from_rdd(training_rdd, names=["features", "labels"], shapes=[[28, 28, 1], []], types=[tf.float32, tf.int32], batch_size=280, val_rdd=testing_rdd) # construct the model from TFDataset images, labels = dataset.tensors with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) # create a optimizer optimizer = TFOptimizer(loss, Adam(1e-3), val_outputs=[logits], val_labels=[labels], val_method=Top1Accuracy()) optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) # kick off training optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) saver = tf.train.Saver() saver.save(optimizer.sess, "/tmp/lenet/")
def main(max_epoch, data_num): sc = init_nncontext() # get data, pre-process and create TFDataset def get_data_rdd(dataset): (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset) image_rdd = sc.parallelize(images_data[:data_num]) labels_rdd = sc.parallelize(labels_data[:data_num]) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) return rdd training_rdd = get_data_rdd("train") testing_rdd = get_data_rdd("test") dataset = TFDataset.from_rdd(training_rdd, names=["features", "labels"], shapes=[[28, 28, 1], []], types=[tf.float32, tf.int32], batch_size=280, val_rdd=testing_rdd ) data = Input(shape=[28, 28, 1]) x = Flatten()(data) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) predictions = Dense(10, activation='softmax')(x) model = Model(input=data, output=predictions) model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy']) optimizer = TFOptimizer.from_keras(model, dataset) optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) # kick off training optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) saver = tf.train.Saver() saver.save(optimizer.sess, "/tmp/lenet/")
def test_tfdataset_with_tfrecord(self): train_path = os.path.join(resource_path, "tfrecord/mnist_train.tfrecord") test_path = os.path.join(resource_path, "tfrecord/mnist_test.tfrecord") dataset = TFDataset.from_tfrecord_file(self.sc, train_path, batch_size=8, validation_file_path=test_path) dataset = dataset.map(lambda x: parse_fn(x[0])) flat = tf.layers.flatten(dataset.feature_tensors) logits = tf.layers.dense(flat, 10) labels = dataset.label_tensors loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) opt = TFOptimizer.from_loss(loss, Adam()) opt.optimize()
def test_tf_optimizer_metrics(self): features = np.random.randn(20, 10) labels = np.random.randint(0, 10, size=[20]) with tf.Graph().as_default(): dataset = TFDataset.from_ndarrays((features, labels), batch_size=4, val_tensors=(features, labels)) feature_tensor, label_tensor = dataset.tensors output = tf.layers.dense(feature_tensor, 10) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=output, labels=label_tensor)) optimizer = TFOptimizer.from_loss(loss, Adam(1e-3), val_outputs=[output], val_labels=[label_tensor], val_method=Accuracy(), metrics={"loss": loss}) optimizer.optimize(end_trigger=MaxEpoch(1)) optimizer.sess.close()
def test_tf_optimizer_metrics(self): features = np.random.randn(20, 10) labels = np.random.randint(0, 10, size=[20]) with tf.Graph().as_default(): dataset = TFDataset.from_ndarrays((features, labels), batch_size=4, val_tensors=(features, labels)) feature_tensor, label_tensor = dataset.tensors features = tf.layers.dense(feature_tensor, 8) output = tf.layers.dense(features, 10) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=output, labels=label_tensor)) optimizer = TFOptimizer.from_loss(loss, { "dense/": Adam(1e-3), "dense_1/": SGD(0.0) }, val_outputs=[output], val_labels=[label_tensor], val_method=Accuracy(), metrics={"loss": loss}) initial_weights = optimizer.tf_model.training_helper_layer.get_weights( ) optimizer.optimize(end_trigger=MaxEpoch(1)) updated_weights = optimizer.tf_model.training_helper_layer.get_weights( ) for i in [ 0, 1 ]: # weights and bias combined with "dense/" should be updated assert not np.allclose(initial_weights[i], updated_weights[i]) for i in [ 2, 3 ]: # weights and bias combined with "dense_1" should be unchanged assert np.allclose(initial_weights[i], updated_weights[i]) optimizer.sess.close()
def main(): sc = init_nncontext() # get data, pre-process and create TFDataset (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "train") image_rdd = sc.parallelize(images_data) labels_rdd = sc.parallelize(labels_data) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) dataset = TFDataset.from_rdd(rdd, names=["features", "labels"], shapes=[[28, 28, 1], [1]], types=[tf.float32, tf.int32], batch_size=280) # construct the model from TFDataset images, labels = dataset.tensors labels = tf.squeeze(labels) with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) # create a optimizer optimizer = TFOptimizer(loss, Adam(1e-3)) optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) # kick off training for i in range(5): optimizer.optimize(end_trigger=MaxEpoch(i + 1)) saver = tf.train.Saver() saver.save(optimizer.sess, "/tmp/lenet/")
def _fit_distributed(self, dataset, validation_split, epochs, **kwargs): self.tf_optimizer = TFOptimizer.from_keras(self.model, dataset, val_spilt=validation_split, **kwargs) self.tf_optimizer.optimize(MaxEpoch(epochs))
output_dim=FC_LINEAR_DIMENSION, # 尺寸: 32 -> 64. activation="sigmoid")) # BigDL 不支持 parameter sharing, 不得已而为之. both_feature = TimeDistributed(layer=convolve_net, input_shape=input_shape)(both_input) encode_left = both_feature.index_select(1, 0) encode_right = both_feature.index_select(1, 1) distance = autograd.abs(encode_left - encode_right) predict = Dense(output_dim=NUM_CLASS_LABEL, activation="sigmoid")(distance) siamese_net = Model(input=both_input, output=predict) siamese_net.compile(optimizer="adam", loss='sparse_categorical_crossentropy', metrics=["accuracy"]) # 构造分布式的数据集对象. data_set = TFDataset.from_rdd(train_rdd, shapes=[input_shape, [1]], batch_size=args.batch_size, val_rdd=test_rdd) optimizer = TFOptimizer.from_keras(siamese_net, data_set) app_name = "Siamese Network" optimizer.set_train_summary(TrainSummary("tmp", app_name)) optimizer.set_val_summary(ValidationSummary("tmp", app_name)) optimizer.optimize(end_trigger=MaxEpoch(args.num_epoch))