def make_input_fns(): train, eval = mnist.load_data() fns = [] for data in (train, eval): x, y = data x = (x.astype(np.float32) * 2. / 255.) - 1. y = y.astype(np.int64) fns.append(tf.estimator.inputs.numpy_input_fn( {'x': x}, y, batch_size=tf.flags.FLAGS.batch_size, num_epochs=None, shuffle=True )) return fns
def onBeginTraining(self): ue.log("starting mnist keras cnn training") model_file_name = "mnistKerasCNN" model_directory = ue.get_content_dir() + "/Scripts/" model_sess_path = model_directory + model_file_name + ".tfsess" model_json_path = model_directory + model_file_name + ".json" my_file = Path(model_json_path) #reset the session each time we get training calls K.clear_session() #let's train batch_size = 128 num_classes = 10 epochs = 8 # input image dimensions img_rows, img_cols = 28, 28 # the data, shuffled and split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() if K.image_data_format() == 'channels_first': x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols) x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols) input_shape = (1, img_rows, img_cols) else: x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 ue.log('x_train shape:' + str(x_train.shape)) ue.log(str(x_train.shape[0]) + 'train samples') ue.log(str(x_test.shape[0]) + 'test samples') # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) # model.add(Dropout(0.2)) # model.add(Flatten()) # model.add(Dense(512, activation='relu')) # model.add(Dropout(0.2)) # model.add(Dense(num_classes, activation='softmax')) #model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test), callbacks=[self.stopcallback]) score = model.evaluate(x_test, y_test, verbose=0) ue.log("mnist keras cnn training complete.") ue.log('Test loss:' + str(score[0])) ue.log('Test accuracy:' + str(score[1])) self.session = K.get_session() self.model = model stored = {'model':model, 'session': self.session} #run a test evaluation ue.log(x_test.shape) result_test = model.predict(np.reshape(x_test[500],(1,28,28,1))) ue.log(result_test) #flush the architecture model data to disk #with open(model_json_path, "w") as json_file: # json_file.write(model.to_json()) #flush the whole model and weights to disk #saver = tf.train.Saver() #save_path = saver.save(K.get_session(), model_sess_path) #model.save(model_path) return stored
from tensorflow.contrib.keras.api.keras.models import Model from tensorflow.contrib.keras.api.keras.layers import Input, Dense from tensorflow.contrib.keras.python.keras.layers import TimeDistributed from tensorflow.contrib.keras.api.keras.layers import LSTM # Training parameters. batch_size = 32 num_classes = 10 epochs = 5 # Embedding dimensions. row_hidden = 128 col_hidden = 128 # The data, shuffled and split between train and test sets. (x_train, y_train), (x_test, y_test) = mnist.load_data() # Reshapes data to 4D for Hierarchical RNN. x_train = x_train.reshape(x_train.shape[0], 28, 28, 1) x_test = x_test.reshape(x_test.shape[0], 28, 28, 1) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 print('x_train shape:', x_train.shape) print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # Converts class vectors to binary class matrices. y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes)
def make_input_fns(): train, test = mnist.load_data() return make_input_fn(train), make_input_fn(test)
def main_fun(args, ctx): import numpy import os import tensorflow as tf import tensorflow.contrib.keras as keras from tensorflow.contrib.keras.api.keras import backend as K from tensorflow.contrib.keras.api.keras.models import Sequential, load_model, save_model from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout from tensorflow.contrib.keras.api.keras.optimizers import RMSprop from tensorflow.contrib.keras.python.keras.callbacks import LambdaCallback, TensorBoard from tensorflow.python.saved_model import builder as saved_model_builder from tensorflow.python.saved_model import tag_constants from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def from tensorflowonspark import TFNode cluster, server = TFNode.start_cluster_server(ctx) if ctx.job_name == "ps": server.join() elif ctx.job_name == "worker": def generate_rdd_data(tf_feed, batch_size): print("generate_rdd_data invoked") while True: batch = tf_feed.next_batch(batch_size) imgs = [] lbls = [] for item in batch: imgs.append(item[0]) lbls.append(item[1]) images = numpy.array(imgs).astype('float32') / 255 labels = numpy.array(lbls).astype('float32') yield (images, labels) with tf.device( tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % ctx.task_index, cluster=cluster)): IMAGE_PIXELS = 28 batch_size = 100 num_classes = 10 # the data, shuffled and split between train and test sets if args.input_mode == 'tf': from tensorflow.contrib.keras.api.keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(60000, 784) x_test = x_test.reshape(10000, 784) x_train = x_train.astype('float32') / 255 x_test = x_test.astype('float32') / 255 # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) else: # args.mode == 'spark' x_train = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS], name="x_train") y_train = tf.placeholder(tf.float32, [None, 10], name="y_train") model = Sequential() model.add(Dense(512, activation='relu', input_shape=(784, ))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(10, activation='softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) saver = tf.train.Saver() with tf.Session(server.target) as sess: K.set_session(sess) def save_checkpoint(epoch, logs=None): if epoch == 1: tf.train.write_graph(sess.graph.as_graph_def(), args.model_dir, 'graph.pbtxt') saver.save(sess, os.path.join(args.model_dir, 'model.ckpt'), global_step=epoch * args.steps_per_epoch) ckpt_callback = LambdaCallback(on_epoch_end=save_checkpoint) tb_callback = TensorBoard(log_dir=args.model_dir, histogram_freq=1, write_graph=True, write_images=True) # add callbacks to save model checkpoint and tensorboard events (on worker:0 only) callbacks = [ckpt_callback, tb_callback ] if ctx.task_index == 0 else None if args.input_mode == 'tf': # train & validate on in-memory data history = model.fit(x_train, y_train, batch_size=batch_size, epochs=args.epochs, verbose=1, validation_data=(x_test, y_test), callbacks=callbacks) else: # args.input_mode == 'spark': # train on data read from a generator which is producing data from a Spark RDD tf_feed = TFNode.DataFeed(ctx.mgr) history = model.fit_generator( generator=generate_rdd_data(tf_feed, batch_size), steps_per_epoch=args.steps_per_epoch, epochs=args.epochs, verbose=1, callbacks=callbacks) if args.export_dir and ctx.job_name == 'worker' and ctx.task_index == 0: # save a local Keras model, so we can reload it with an inferencing learning_phase save_model(model, "tmp_model") # reload the model K.set_learning_phase(False) new_model = load_model("tmp_model") # export a saved_model for inferencing builder = saved_model_builder.SavedModelBuilder( args.export_dir) signature = predict_signature_def( inputs={'images': new_model.input}, outputs={'scores': new_model.output}) builder.add_meta_graph_and_variables( sess=sess, tags=[tag_constants.SERVING], signature_def_map={'predict': signature}, clear_devices=True) builder.save() if args.input_mode == 'spark': tf_feed.terminate()
from tensorflow.contrib.keras.api.keras.models import Sequential from tensorflow.contrib.keras.api.keras.layers import Dense, Activation, Convolution2D # from tensorflow.examples.tutorials.mnist import input_data from tensorflow.python.keras._impl.keras.utils import np_utils # mnist = input_data.read_data_sets('MNIST_data', one_hot=True) batch_size = 128 n_classes = 10 n_epoch = 12 # 训练轮数 img_rows, img_cols = 28, 28 n_filters = 32 poll_size = (2, 2) # 池化大小 kernel_size = (3, 3) # 卷积核大小 (train_x, train_y), (test_x, test_y) = mnist.load_data() train_x = train_x.reshape(train_x.shape[0], img_rows, img_cols, 1) test_x = test_x.reshape(test_x.shape[0], img_rows, img_cols, 1) input_shape = (img_rows, img_cols, 1) train_x = train_x.astype('float32') test_x = test_x.astype('float32') train_x /= 255 test_x /= 255 # 将类向量转换成二进制矩阵 train_y = np_utils.to_categorical(train_y, n_classes) test_y = np_utils.to_categorical(test_y, n_classes) # 构建训练模型 model = Sequential() # 添加卷积层 model.add(Convolution2D(n_filters, kernel_size[0],
def load_dataset(): train, test = mnist.load_data() return train, test