def build_generator_dense():
model = Sequential()
# Add arbitrary layers
first = True
for size in generator_layers.split(":"):
size = int(size)
if first:
model.add(Dense(size, input_shape=noise_shape, activation=generator_activation))
else:
model.add(Dense(size, activation=generator_activation))
model.add(Dropout(dropout_value))
first = False
# Add the final layer
model.add(Dense( np.prod(url_shape) , activation="tanh"))
model.add(Dropout(dropout_value))
model.add(Reshape(url_shape))
model.summary()
# Build the model
noise = Input(shape=noise_shape)
gen = model(noise)
return Model(noise, gen)
def train(data,
file_name,
params,
num_epochs=50,
batch_size=128,
train_temp=1,
init=None,
lr=0.01,
decay=1e-5,
momentum=0.9):
"""
Train a n-layer simple network for MNIST and CIFAR
"""
# create a Keras sequential model
model = Sequential()
# reshape the input (28*28*1) or (32*32*3) to 1-D
model.add(Flatten(input_shape=data.train_data.shape[1:]))
# dense layers (the hidden layer)
for param in params:
model.add(Dense(param))
# ReLU activation
model.add(Activation('relu'))
# the output layer, with 10 classes
model.add(Dense(10))
# load initial weights when given
if init != None:
model.load_weights(init)
# define the loss function which is the cross entropy between prediction and true label
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted /
train_temp)
# initiate the SGD optimizer with given hyper parameters
sgd = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True)
# compile the Keras model, given the specified loss and optimizer
model.compile(loss=fn, optimizer=sgd, metrics=['accuracy'])
model.summary()
print("Traing a {} layer model, saving to {}".format(
len(params) + 1, file_name))
# run training with given dataset, and print progress
history = model.fit(data.train_data,
data.train_labels,
batch_size=batch_size,
validation_data=(data.validation_data,
data.validation_labels),
epochs=num_epochs,
shuffle=True)
# save model to a file
if file_name != None:
model.save(file_name)
return {'model': model, 'history': history}
def train(data, file_name, params, num_epochs=50, batch_size=256, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9, activation="relu", optimizer_name="sgd"):
"""
Train a n-layer simple network for MNIST and CIFAR
"""
# create a Keras sequential model
model = Sequential()
# reshape the input (28*28*1) or (32*32*3) to 1-D
model.add(Flatten(input_shape=data.train_data.shape[1:]))
# dense layers (the hidden layer)
n = 0
for param in params:
n += 1
model.add(Dense(param, kernel_initializer='he_uniform'))
# ReLU activation
if activation == "arctan":
model.add(Lambda(lambda x: tf.atan(x), name=activation+"_"+str(n)))
else:
model.add(Activation(activation, name=activation+"_"+str(n)))
# the output layer, with 10 classes
model.add(Dense(10, kernel_initializer='he_uniform'))
# load initial weights when given
if init != None:
model.load_weights(init)
# define the loss function which is the cross entropy between prediction and true label
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted/train_temp)
if optimizer_name == "sgd":
# initiate the SGD optimizer with given hyper parameters
optimizer = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True)
elif optimizer_name == "adam":
optimizer = Adam(lr=lr, beta_1 = 0.9, beta_2 = 0.999, epsilon = None, decay=decay, amsgrad=False)
# compile the Keras model, given the specified loss and optimizer
model.compile(loss=fn,
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
print("Traing a {} layer model, saving to {}".format(len(params) + 1, file_name))
# run training with given dataset, and print progress
history = model.fit(data.train_data, data.train_labels,
batch_size=batch_size,
validation_data=(data.validation_data, data.validation_labels),
epochs=num_epochs,
shuffle=True)
# save model to a file
if file_name != None:
model.save(file_name)
print('model saved to ', file_name)
return {'model':model, 'history':history}
def build_discriminator_dense():
model = Sequential()
model.add(Flatten(input_shape=url_shape))
# Add arbitrary layers
for size in discriminator_layers.split(":"):
size = int(size)
model.add(Dense(size, activation=discriminator_activation))
model.add(Dropout(dropout_value))
# Add the final layer, with a single output
model.add(Dense(1, activation='sigmoid'))
model.summary()
# Build the model
gen = Input(shape=url_shape)
validity = model(gen)
return Model(gen, validity)
def __init__(self, img_size, num_channels, compress_mode=1, resize=None):
self.compress_mode = compress_mode
working_img_size = img_size
encoder_model = Sequential()
# resize input to a size easy for down-sampled
if resize:
encoder_model.add(
Lambda(lambda image: tf.image.resize_images(
image, (resize, resize)),
input_shape=(img_size, img_size, num_channels)))
else:
encoder_model.add(
Convolution2D(16,
3,
strides=1,
padding='same',
input_shape=(img_size, img_size, num_channels)))
BatchNormalization(axis=3)
encoder_model.add(Activation("relu"))
encoder_model.add(MaxPooling2D(pool_size=2, strides=2, padding='same'))
working_img_size //= 2
if compress_mode >= 2:
encoder_model.add(Convolution2D(16, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
encoder_model.add(Activation("relu"))
encoder_model.add(
MaxPooling2D(pool_size=2, strides=2, padding='same'))
working_img_size //= 2
if compress_mode >= 3:
encoder_model.add(Convolution2D(16, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
encoder_model.add(Activation("relu"))
encoder_model.add(
MaxPooling2D(pool_size=2, strides=2, padding='same'))
working_img_size //= 2
encoder_model.add(
Convolution2D(num_channels, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
decoder_model = Sequential()
decoder_model.add(encoder_model)
if compress_mode >= 3:
working_img_size *= 2
decoder_model.add(Convolution2D(16, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
decoder_model.add(Activation("relu"))
#decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (working_img_size, working_img_size))))
decoder_model.add(UpSampling2D((2, 2),
data_format='channels_last'))
if compress_mode >= 2:
working_img_size *= 2
decoder_model.add(Convolution2D(16, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
decoder_model.add(Activation("relu"))
#decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (working_img_size, working_img_size))))
decoder_model.add(UpSampling2D((2, 2),
data_format='channels_last'))
working_img_size *= 2
decoder_model.add(Convolution2D(16, 3, strides=1, padding='same'))
BatchNormalization(axis=3)
decoder_model.add(Activation("relu"))
# decoder_model.add(Lambda(lambda image: tf.image.resize_images(image, (img_size, img_size))))
decoder_model.add(UpSampling2D((2, 2), data_format='channels_last'))
if resize:
decoder_model.add(
Lambda(lambda image: tf.image.resize_images(
image, (img_size, img_size))))
decoder_model.add(
Convolution2D(num_channels, 3, strides=1, padding='same'))
# decoder_model.add(Lambda(lambda image: K.clip(image, -clip_value, clip_value) ))
print('Encoder model:')
encoder_model.summary()
print('Decoder model:')
decoder_model.summary()
self.encoder = encoder_model
self.decoder = decoder_model
def main_fun(args, ctx):
import numpy
import os
import tensorflow as tf
import tensorflow.contrib.keras as keras
from tensorflow.contrib.keras.api.keras import backend as K
from tensorflow.contrib.keras.api.keras.models import Sequential, load_model, save_model
from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout
from tensorflow.contrib.keras.api.keras.optimizers import RMSprop
from tensorflow.contrib.keras.python.keras.callbacks import LambdaCallback, TensorBoard
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model import tag_constants
from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def
from tensorflowonspark import TFNode
cluster, server = TFNode.start_cluster_server(ctx)
if ctx.job_name == "ps":
server.join()
elif ctx.job_name == "worker":
def generate_rdd_data(tf_feed, batch_size):
print("generate_rdd_data invoked")
while True:
batch = tf_feed.next_batch(batch_size)
imgs = []
lbls = []
for item in batch:
imgs.append(item[0])
lbls.append(item[1])
images = numpy.array(imgs).astype('float32') / 255
labels = numpy.array(lbls).astype('float32')
yield (images, labels)
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % ctx.task_index,
cluster=cluster)):
IMAGE_PIXELS = 28
batch_size = 100
num_classes = 10
# the data, shuffled and split between train and test sets
if args.input_mode == 'tf':
from tensorflow.contrib.keras.api.keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
else: # args.mode == 'spark'
x_train = tf.placeholder(tf.float32,
[None, IMAGE_PIXELS * IMAGE_PIXELS],
name="x_train")
y_train = tf.placeholder(tf.float32, [None, 10],
name="y_train")
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784, )))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
saver = tf.train.Saver()
with tf.Session(server.target) as sess:
K.set_session(sess)
def save_checkpoint(epoch, logs=None):
if epoch == 1:
tf.train.write_graph(sess.graph.as_graph_def(),
args.model_dir, 'graph.pbtxt')
saver.save(sess,
os.path.join(args.model_dir, 'model.ckpt'),
global_step=epoch * args.steps_per_epoch)
ckpt_callback = LambdaCallback(on_epoch_end=save_checkpoint)
tb_callback = TensorBoard(log_dir=args.model_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
# add callbacks to save model checkpoint and tensorboard events (on worker:0 only)
callbacks = [ckpt_callback, tb_callback
] if ctx.task_index == 0 else None
if args.input_mode == 'tf':
# train & validate on in-memory data
history = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=args.epochs,
verbose=1,
validation_data=(x_test, y_test),
callbacks=callbacks)
else: # args.input_mode == 'spark':
# train on data read from a generator which is producing data from a Spark RDD
tf_feed = TFNode.DataFeed(ctx.mgr)
history = model.fit_generator(
generator=generate_rdd_data(tf_feed, batch_size),
steps_per_epoch=args.steps_per_epoch,
epochs=args.epochs,
verbose=1,
callbacks=callbacks)
if args.export_dir and ctx.job_name == 'worker' and ctx.task_index == 0:
# save a local Keras model, so we can reload it with an inferencing learning_phase
save_model(model, "tmp_model")
# reload the model
K.set_learning_phase(False)
new_model = load_model("tmp_model")
# export a saved_model for inferencing
builder = saved_model_builder.SavedModelBuilder(
args.export_dir)
signature = predict_signature_def(
inputs={'images': new_model.input},
outputs={'scores': new_model.output})
builder.add_meta_graph_and_variables(
sess=sess,
tags=[tag_constants.SERVING],
signature_def_map={'predict': signature},
clear_devices=True)
builder.save()
if args.input_mode == 'spark':
tf_feed.terminate()
def train_cnn_7layer(data,
file_name,
params,
num_epochs=50,
batch_size=256,
train_temp=1,
init=None,
lr=0.01,
decay=1e-5,
momentum=0.9,
activation="relu",
optimizer_name="sgd"):
"""
Train a 7-layer cnn network for MNIST and CIFAR (same as the cnn model in Clever)
mnist: 32 32 64 64 200 200
cifar: 64 64 128 128 256 256
"""
# create a Keras sequential model
model = Sequential()
print("training data shape = {}".format(data.train_data.shape))
# define model structure
model.add(Conv2D(params[0], (3, 3), input_shape=data.train_data.shape[1:]))
model.add(Activation(activation))
model.add(Conv2D(params[1], (3, 3)))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(params[2], (3, 3)))
model.add(Activation(activation))
model.add(Conv2D(params[3], (3, 3)))
model.add(Activation(activation))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(params[4]))
model.add(Activation(activation))
model.add(Dropout(0.5))
model.add(Dense(params[5]))
model.add(Activation(activation))
model.add(Dense(10))
# load initial weights when given
if init != None:
model.load_weights(init)
# define the loss function which is the cross entropy between prediction and true label
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted /
train_temp)
if optimizer_name == "sgd":
# initiate the SGD optimizer with given hyper parameters
optimizer = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True)
elif optimizer_name == "adam":
optimizer = Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=decay,
amsgrad=False)
# compile the Keras model, given the specified loss and optimizer
model.compile(loss=fn, optimizer=optimizer, metrics=['accuracy'])
model.summary()
print("Traing a {} layer model, saving to {}".format(
len(params) + 1, file_name))
# run training with given dataset, and print progress
history = model.fit(data.train_data,
data.train_labels,
batch_size=batch_size,
validation_data=(data.validation_data,
data.validation_labels),
epochs=num_epochs,
shuffle=True)
# save model to a file
if file_name != None:
model.save(file_name)
print('model saved to ', file_name)
return {'model': model, 'history': history}
def train(data,
file_name,
filters,
kernels,
num_epochs=50,
batch_size=128,
train_temp=1,
init=None,
activation=tf.nn.relu,
bn=False):
"""
Train a n-layer CNN for MNIST and CIFAR
"""
# create a Keras sequential model
model = Sequential()
model.add(
Conv2D(filters[0], kernels[0], input_shape=data.train_data.shape[1:]))
if bn:
model.add(BatchNormalization())
model.add(Lambda(activation))
for f, k in zip(filters[1:], kernels[1:]):
model.add(Conv2D(f, k))
if bn:
model.add(BatchNormalization())
# ReLU activation
model.add(Lambda(activation))
# the output layer, with 10 classes
model.add(Flatten())
model.add(Dense(10))
# load initial weights when given
if init != None:
model.load_weights(init)
# define the loss function which is the cross entropy between prediction and true label
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted /
train_temp)
# initiate the Adam optimizer
sgd = Adam()
# compile the Keras model, given the specified loss and optimizer
model.compile(loss=fn, optimizer=sgd, metrics=['accuracy'])
model.summary()
print("Traing a {} layer model, saving to {}".format(
len(filters) + 1, file_name))
# run training with given dataset, and print progress
history = model.fit(data.train_data,
data.train_labels,
batch_size=batch_size,
validation_data=(data.validation_data,
data.validation_labels),
epochs=num_epochs,
shuffle=True)
# save model to a file
if file_name != None:
model.save(file_name)
return {'model': model, 'history': history}
class TinyYoloV2:
"""
This class handles the building and the loss of the
tiny yolo v2 network.
"""
def __init__(self, config):
"""
Initializes class variables.
:param config: Contains the networks hyperparameters
"""
#super.__init__(self)
self.config = config
self.network = None
self.loss = None
self.input_shape = config.input_shape
def build(self):
"""
Builds the tiny yolo v2 network.
:param input: input image batch to the network
:return: logits output from network
"""
self.model = Sequential()
self.model.add(Convolution2D(16, (3, 3), input_shape=(416, 416, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.model.add(Convolution2D(32, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.model.add(Convolution2D(64, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.model.add(Convolution2D(128, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.model.add(Convolution2D(256, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
self.model.add(Convolution2D(512, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 1), padding='valid'))
self.model.add(Convolution2D(1024, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(Convolution2D(1024, (3, 3), padding='same'))
self.model.add(LeakyReLU())
self.model.add(BatchNormalization())
self.model.add(Convolution2D(125, (1, 1), activation=None))
if self.config.optimizer == 'adam':
opt = Adam()
elif self.config.optimizer == 'sgd':
opt = SGD()
if self.config.loss == 'categorical_crossentropy':
loss = 'categorical_crossentropy'
elif self.config.loss == 'yolov2_loss':
raise NotImplemented
self.model.compile(loss=loss, optimizer=opt, metrics=['accuracy'])
self.model.summary()
return self.model
def convertToBoxParams(self, out):
"""
Convert the final layer features to bounding box parameters.
:return: box_xy: tensor
x, y box predictions adjusted by spatial location in conv layer
box_wh: tensor
w, h box predictions adjusted by anchors and conv spatial resolution
box_conf: tensor
Probability estimate for whether each box contains any object
box_class_pred : tensor
Probability distribution estimate for each box over class labels
"""
feats = out
num_anchors = len(self.config.anchors)
# Reshape to batch, height, width, num_anchors, box_params
anchors_tensor = K.reshape(K.variable(self.config.anchors), [1, 1, 1, num_anchors, 2])
conv_dims = K.shape(feats)[1:3] # assuming channels last
# In YOLO the height index is the inner most iteration
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
conv_width_index = K.tile(
K.expand_dims(conv_width_index, 0), [conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats, [-1, conv_dims[0], conv_dims[1], num_anchors, self.config.classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))
# Static generation of conv_index:
# conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)])
# conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering.
# conv_index = K.variable(
# conv_index.reshape(1, conv_height, conv_width, 1, 2))
# feats = Reshape(
# (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats)
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.softmax(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
# Note: YOLO iterates over height index before width index.
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_xy, box_wh, box_confidence, box_class_probs
def save_model(self, name):
"""
Saves model to path.
:return:
"""
path = "C:\ObjectDetection\Data\ModelCheckpoints" + name + ".h5"
self.model.save(path)
def save_weights(self, name):
"""
Saves the model weights to path.
:param name:
:return:
"""
path = "C:\ObjectDetection\Data\ModelCheckpoints" + name + ".h5"
self.model.save_weights(path)
def load_weights(self):
print("About to load weights.")
utils.load_weights(self.model, 'C:\\ObjectDetection\\Data\\ModelCheckpoints\\tiny-yolo-voc.weights')
print("Loaded weights.")
def _load_pretrained_network(self):
"""
Loads the pretrained network's weights
into the new network.
:return:
"""
raise NotImplemented
