def build_generator_dense(): model = Sequential() # Add arbitrary layers first = True for size in generator_layers.split(":"): size = int(size) if first: model.add(Dense(size, input_shape=noise_shape, activation=generator_activation)) else: model.add(Dense(size, activation=generator_activation)) model.add(Dropout(dropout_value)) first = False # Add the final layer model.add(Dense( np.prod(url_shape) , activation="tanh")) model.add(Dropout(dropout_value)) model.add(Reshape(url_shape)) model.summary() # Build the model noise = Input(shape=noise_shape) gen = model(noise) return Model(noise, gen)
def train(data, file_name, params, num_epochs=50, batch_size=128, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9): """ Train a n-layer simple network for MNIST and CIFAR """ # create a Keras sequential model model = Sequential() # reshape the input (28*28*1) or (32*32*3) to 1-D model.add(Flatten(input_shape=data.train_data.shape[1:])) # dense layers (the hidden layer) for param in params: model.add(Dense(param)) # ReLU activation model.add(Activation('relu')) # the output layer, with 10 classes model.add(Dense(10)) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) # initiate the SGD optimizer with given hyper parameters sgd = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.summary() print("Traing a {} layer model, saving to {}".format( len(params) + 1, file_name)) # run training with given dataset, and print progress history = model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) return {'model': model, 'history': history}
def train(data, file_name, params, num_epochs=50, batch_size=256, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9, activation="relu", optimizer_name="sgd"): """ Train a n-layer simple network for MNIST and CIFAR """ # create a Keras sequential model model = Sequential() # reshape the input (28*28*1) or (32*32*3) to 1-D model.add(Flatten(input_shape=data.train_data.shape[1:])) # dense layers (the hidden layer) n = 0 for param in params: n += 1 model.add(Dense(param, kernel_initializer='he_uniform')) # ReLU activation if activation == "arctan": model.add(Lambda(lambda x: tf.atan(x), name=activation+"_"+str(n))) else: model.add(Activation(activation, name=activation+"_"+str(n))) # the output layer, with 10 classes model.add(Dense(10, kernel_initializer='he_uniform')) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted/train_temp) if optimizer_name == "sgd": # initiate the SGD optimizer with given hyper parameters optimizer = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) elif optimizer_name == "adam": optimizer = Adam(lr=lr, beta_1 = 0.9, beta_2 = 0.999, epsilon = None, decay=decay, amsgrad=False) # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=optimizer, metrics=['accuracy']) model.summary() print("Traing a {} layer model, saving to {}".format(len(params) + 1, file_name)) # run training with given dataset, and print progress history = model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) print('model saved to ', file_name) return {'model':model, 'history':history}
def build_discriminator_dense(): model = Sequential() model.add(Flatten(input_shape=url_shape)) # Add arbitrary layers for size in discriminator_layers.split(":"): size = int(size) model.add(Dense(size, activation=discriminator_activation)) model.add(Dropout(dropout_value)) # Add the final layer, with a single output model.add(Dense(1, activation='sigmoid')) model.summary() # Build the model gen = Input(shape=url_shape) validity = model(gen) return Model(gen, validity)
def __init__(self, img_size, num_channels, compress_mode=1, resize=None): self.compress_mode = compress_mode working_img_size = img_size encoder_model = Sequential() # resize input to a size easy for down-sampled if resize: encoder_model.add( Lambda(lambda image: tf.image.resize_images( image, (resize, resize)), input_shape=(img_size, img_size, num_channels))) else: encoder_model.add( Convolution2D(16, 3, strides=1, padding='same', input_shape=(img_size, img_size, num_channels))) BatchNormalization(axis=3) encoder_model.add(Activation("relu")) encoder_model.add(MaxPooling2D(pool_size=2, strides=2, padding='same')) working_img_size //= 2 if compress_mode >= 2: encoder_model.add(Convolution2D(16, 3, strides=1, padding='same')) BatchNormalization(axis=3) encoder_model.add(Activation("relu")) encoder_model.add( MaxPooling2D(pool_size=2, strides=2, padding='same')) working_img_size //= 2 if compress_mode >= 3: encoder_model.add(Convolution2D(16, 3, strides=1, padding='same')) BatchNormalization(axis=3) encoder_model.add(Activation("relu")) encoder_model.add( MaxPooling2D(pool_size=2, strides=2, padding='same')) working_img_size //= 2 encoder_model.add( Convolution2D(num_channels, 3, strides=1, padding='same')) BatchNormalization(axis=3) decoder_model = Sequential() decoder_model.add(encoder_model) if compress_mode >= 3: working_img_size *= 2 decoder_model.add(Convolution2D(16, 3, strides=1, padding='same')) BatchNormalization(axis=3) decoder_model.add(Activation("relu")) #decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (working_img_size, working_img_size)))) decoder_model.add(UpSampling2D((2, 2), data_format='channels_last')) if compress_mode >= 2: working_img_size *= 2 decoder_model.add(Convolution2D(16, 3, strides=1, padding='same')) BatchNormalization(axis=3) decoder_model.add(Activation("relu")) #decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (working_img_size, working_img_size)))) decoder_model.add(UpSampling2D((2, 2), data_format='channels_last')) working_img_size *= 2 decoder_model.add(Convolution2D(16, 3, strides=1, padding='same')) BatchNormalization(axis=3) decoder_model.add(Activation("relu")) # decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (img_size, img_size)))) decoder_model.add(UpSampling2D((2, 2), data_format='channels_last')) if resize: decoder_model.add( Lambda(lambda image: tf.image.resize_images( image, (img_size, img_size)))) decoder_model.add( Convolution2D(num_channels, 3, strides=1, padding='same')) # decoder_model.add(Lambda(lambda image: K.clip(image, -clip_value, clip_value) )) print('Encoder model:') encoder_model.summary() print('Decoder model:') decoder_model.summary() self.encoder = encoder_model self.decoder = decoder_model
def main_fun(args, ctx): import numpy import os import tensorflow as tf import tensorflow.contrib.keras as keras from tensorflow.contrib.keras.api.keras import backend as K from tensorflow.contrib.keras.api.keras.models import Sequential, load_model, save_model from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout from tensorflow.contrib.keras.api.keras.optimizers import RMSprop from tensorflow.contrib.keras.python.keras.callbacks import LambdaCallback, TensorBoard from tensorflow.python.saved_model import builder as saved_model_builder from tensorflow.python.saved_model import tag_constants from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def from tensorflowonspark import TFNode cluster, server = TFNode.start_cluster_server(ctx) if ctx.job_name == "ps": server.join() elif ctx.job_name == "worker": def generate_rdd_data(tf_feed, batch_size): print("generate_rdd_data invoked") while True: batch = tf_feed.next_batch(batch_size) imgs = [] lbls = [] for item in batch: imgs.append(item[0]) lbls.append(item[1]) images = numpy.array(imgs).astype('float32') / 255 labels = numpy.array(lbls).astype('float32') yield (images, labels) with tf.device( tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % ctx.task_index, cluster=cluster)): IMAGE_PIXELS = 28 batch_size = 100 num_classes = 10 # the data, shuffled and split between train and test sets if args.input_mode == 'tf': from tensorflow.contrib.keras.api.keras.datasets import mnist (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape(60000, 784) x_test = x_test.reshape(10000, 784) x_train = x_train.astype('float32') / 255 x_test = x_test.astype('float32') / 255 # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) else: # args.mode == 'spark' x_train = tf.placeholder(tf.float32, [None, IMAGE_PIXELS * IMAGE_PIXELS], name="x_train") y_train = tf.placeholder(tf.float32, [None, 10], name="y_train") model = Sequential() model.add(Dense(512, activation='relu', input_shape=(784, ))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(10, activation='softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) saver = tf.train.Saver() with tf.Session(server.target) as sess: K.set_session(sess) def save_checkpoint(epoch, logs=None): if epoch == 1: tf.train.write_graph(sess.graph.as_graph_def(), args.model_dir, 'graph.pbtxt') saver.save(sess, os.path.join(args.model_dir, 'model.ckpt'), global_step=epoch * args.steps_per_epoch) ckpt_callback = LambdaCallback(on_epoch_end=save_checkpoint) tb_callback = TensorBoard(log_dir=args.model_dir, histogram_freq=1, write_graph=True, write_images=True) # add callbacks to save model checkpoint and tensorboard events (on worker:0 only) callbacks = [ckpt_callback, tb_callback ] if ctx.task_index == 0 else None if args.input_mode == 'tf': # train & validate on in-memory data history = model.fit(x_train, y_train, batch_size=batch_size, epochs=args.epochs, verbose=1, validation_data=(x_test, y_test), callbacks=callbacks) else: # args.input_mode == 'spark': # train on data read from a generator which is producing data from a Spark RDD tf_feed = TFNode.DataFeed(ctx.mgr) history = model.fit_generator( generator=generate_rdd_data(tf_feed, batch_size), steps_per_epoch=args.steps_per_epoch, epochs=args.epochs, verbose=1, callbacks=callbacks) if args.export_dir and ctx.job_name == 'worker' and ctx.task_index == 0: # save a local Keras model, so we can reload it with an inferencing learning_phase save_model(model, "tmp_model") # reload the model K.set_learning_phase(False) new_model = load_model("tmp_model") # export a saved_model for inferencing builder = saved_model_builder.SavedModelBuilder( args.export_dir) signature = predict_signature_def( inputs={'images': new_model.input}, outputs={'scores': new_model.output}) builder.add_meta_graph_and_variables( sess=sess, tags=[tag_constants.SERVING], signature_def_map={'predict': signature}, clear_devices=True) builder.save() if args.input_mode == 'spark': tf_feed.terminate()
def train_cnn_7layer(data, file_name, params, num_epochs=50, batch_size=256, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9, activation="relu", optimizer_name="sgd"): """ Train a 7-layer cnn network for MNIST and CIFAR (same as the cnn model in Clever) mnist: 32 32 64 64 200 200 cifar: 64 64 128 128 256 256 """ # create a Keras sequential model model = Sequential() print("training data shape = {}".format(data.train_data.shape)) # define model structure model.add(Conv2D(params[0], (3, 3), input_shape=data.train_data.shape[1:])) model.add(Activation(activation)) model.add(Conv2D(params[1], (3, 3))) model.add(Activation(activation)) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(params[2], (3, 3))) model.add(Activation(activation)) model.add(Conv2D(params[3], (3, 3))) model.add(Activation(activation)) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(params[4])) model.add(Activation(activation)) model.add(Dropout(0.5)) model.add(Dense(params[5])) model.add(Activation(activation)) model.add(Dense(10)) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) if optimizer_name == "sgd": # initiate the SGD optimizer with given hyper parameters optimizer = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) elif optimizer_name == "adam": optimizer = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=decay, amsgrad=False) # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=optimizer, metrics=['accuracy']) model.summary() print("Traing a {} layer model, saving to {}".format( len(params) + 1, file_name)) # run training with given dataset, and print progress history = model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) print('model saved to ', file_name) return {'model': model, 'history': history}
def train(data, file_name, filters, kernels, num_epochs=50, batch_size=128, train_temp=1, init=None, activation=tf.nn.relu, bn=False): """ Train a n-layer CNN for MNIST and CIFAR """ # create a Keras sequential model model = Sequential() model.add( Conv2D(filters[0], kernels[0], input_shape=data.train_data.shape[1:])) if bn: model.add(BatchNormalization()) model.add(Lambda(activation)) for f, k in zip(filters[1:], kernels[1:]): model.add(Conv2D(f, k)) if bn: model.add(BatchNormalization()) # ReLU activation model.add(Lambda(activation)) # the output layer, with 10 classes model.add(Flatten()) model.add(Dense(10)) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) # initiate the Adam optimizer sgd = Adam() # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.summary() print("Traing a {} layer model, saving to {}".format( len(filters) + 1, file_name)) # run training with given dataset, and print progress history = model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) return {'model': model, 'history': history}
class TinyYoloV2: """ This class handles the building and the loss of the tiny yolo v2 network. """ def __init__(self, config): """ Initializes class variables. :param config: Contains the networks hyperparameters """ #super.__init__(self) self.config = config self.network = None self.loss = None self.input_shape = config.input_shape def build(self): """ Builds the tiny yolo v2 network. :param input: input image batch to the network :return: logits output from network """ self.model = Sequential() self.model.add(Convolution2D(16, (3, 3), input_shape=(416, 416, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) self.model.add(Convolution2D(32, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) self.model.add(Convolution2D(64, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) self.model.add(Convolution2D(128, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) self.model.add(Convolution2D(256, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid')) self.model.add(Convolution2D(512, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1), padding='valid')) self.model.add(Convolution2D(1024, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(Convolution2D(1024, (3, 3), padding='same')) self.model.add(LeakyReLU()) self.model.add(BatchNormalization()) self.model.add(Convolution2D(125, (1, 1), activation=None)) if self.config.optimizer == 'adam': opt = Adam() elif self.config.optimizer == 'sgd': opt = SGD() if self.config.loss == 'categorical_crossentropy': loss = 'categorical_crossentropy' elif self.config.loss == 'yolov2_loss': raise NotImplemented self.model.compile(loss=loss, optimizer=opt, metrics=['accuracy']) self.model.summary() return self.model def convertToBoxParams(self, out): """ Convert the final layer features to bounding box parameters. :return: box_xy: tensor x, y box predictions adjusted by spatial location in conv layer box_wh: tensor w, h box predictions adjusted by anchors and conv spatial resolution box_conf: tensor Probability estimate for whether each box contains any object box_class_pred : tensor Probability distribution estimate for each box over class labels """ feats = out num_anchors = len(self.config.anchors) # Reshape to batch, height, width, num_anchors, box_params anchors_tensor = K.reshape(K.variable(self.config.anchors), [1, 1, 1, num_anchors, 2]) conv_dims = K.shape(feats)[1:3] # assuming channels last # In YOLO the height index is the inner most iteration conv_height_index = K.arange(0, stop=conv_dims[0]) conv_width_index = K.arange(0, stop=conv_dims[1]) conv_height_index = K.tile(conv_height_index, [conv_dims[1]]) conv_width_index = K.tile( K.expand_dims(conv_width_index, 0), [conv_dims[0], 1]) conv_width_index = K.flatten(K.transpose(conv_width_index)) conv_index = K.transpose(K.stack([conv_height_index, conv_width_index])) conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2]) conv_index = K.cast(conv_index, K.dtype(feats)) feats = K.reshape( feats, [-1, conv_dims[0], conv_dims[1], num_anchors, self.config.classes + 5]) conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats)) # Static generation of conv_index: # conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)]) # conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering. # conv_index = K.variable( # conv_index.reshape(1, conv_height, conv_width, 1, 2)) # feats = Reshape( # (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats) box_xy = K.sigmoid(feats[..., :2]) box_wh = K.exp(feats[..., 2:4]) box_confidence = K.sigmoid(feats[..., 4:5]) box_class_probs = K.softmax(feats[..., 5:]) # Adjust preditions to each spatial grid point and anchor size. # Note: YOLO iterates over height index before width index. box_xy = (box_xy + conv_index) / conv_dims box_wh = box_wh * anchors_tensor / conv_dims return box_xy, box_wh, box_confidence, box_class_probs def save_model(self, name): """ Saves model to path. :return: """ path = "C:\ObjectDetection\Data\ModelCheckpoints" + name + ".h5" self.model.save(path) def save_weights(self, name): """ Saves the model weights to path. :param name: :return: """ path = "C:\ObjectDetection\Data\ModelCheckpoints" + name + ".h5" self.model.save_weights(path) def load_weights(self): print("About to load weights.") utils.load_weights(self.model, 'C:\\ObjectDetection\\Data\\ModelCheckpoints\\tiny-yolo-voc.weights') print("Loaded weights.") def _load_pretrained_network(self): """ Loads the pretrained network's weights into the new network. :return: """ raise NotImplemented