def __init__(self,
input_shape=(778, 576, 1),
num_categories=5,
parallel_mode=False,
verbose=False):
"""
https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models
https://keras.io/getting-started/functional-api-guide/#shared-layers
https://blog.keras.io/building-autoencoders-in-keras.html
https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py
"""
self.num_categories = num_categories
filters = 128
img_rows, img_cols, img_chns = input_shape
input_img = Input(shape=input_shape, name="main_input")
if verbose:
print("Network input shape is", input_img.get_shape())
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(input_img)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.05)(x)
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.05)(x)
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.05)(x)
self.framer = Model(input_img, x)
optimizer = Nadam(lr=0.0002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004,
clipnorm=0.618)
self.framer.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['acc'])
def super_resolution(input_tensor,
n_feature_layers=2,
n_projection=1,
feature_filters=[128, 32],
projection_filters=32,
k_size=[3, 1],
s=2):
'''
input_tensor: The input tensor required to complete the model
n_features_layer: The number of initial feature extraction layers that needs to be
fixed before starting the up and down projection blocks
n_projection: Number of Up and down projection blocks
feature_filters: Number of filters in each feature extraction layer
k_size: The kernel size to be used in feature extraction layer
s: The scaling factor of the super resolution network
Returns
-------
A Model object of the super resolution network
'''
if not K_B.is_keras_tensor(input_tensor):
input_tensor = Input(shape=input_tensor.get_shape()[1:],
tensor=input_tensor)
x = Conv2D(feature_filters[0], k_size[0], padding="SAME")(input_tensor)
if (len(k_size) != len(feature_filters)):
raise ValueError(
"Number of Layers for feature extraction must be equal to the number of filters sets given."
)
for i in range(len(k_size) - 1):
x = Conv2D(feature_filters[i + 1], k_size[i + 1], padding="same")(x)
for i in range(n_projection):
x = up_projection(x, projection_filters, s, i + 1)
x = down_projection(x, projection_filters, s, i + 1)
x = up_projection(x, projection_filters, s, n_projection + 1)
x = Conv2D(3, 3, padding='same')(x)
return Model(inputs=input_tensor, outputs=x)
def __init__(self,
input_shape=(64, 64, 1),
num_categories=5,
parallel_mode=False,
verbose=False):
"""
https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models
https://keras.io/getting-started/functional-api-guide/#shared-layers
https://blog.keras.io/building-autoencoders-in-keras.html
https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py
"""
self.num_categories = num_categories
# number of filters
# convolution kernel size
filters = 128
latent_dim = 64
epsilon_std = 1.0
noise_std = .01
img_rows, img_cols, img_chns = input_shape
input_img = Input(shape=input_shape, name="main_input")
if verbose:
print("Network input shape is", input_img.get_shape())
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(input_img)
x = LeakyReLU(alpha=0.05)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(x)
x = LeakyReLU(alpha=0.05)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(filters, (3, 3),
padding='same',
activity_regularizer=l2(10e-8))(x)
x = LeakyReLU(alpha=0.05)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same', name="encoded")(x)
conv_output_shape = K.int_shape(x)
intermediate_dim = conv_output_shape[1] * conv_output_shape[
2] * conv_output_shape[3]
if verbose:
print("Convolution output shape is", conv_output_shape)
x = Flatten()(x)
hidden = Dense(intermediate_dim, activation='selu')(x)
z_mean = Dense(latent_dim)(hidden)
z_log_var = Dense(latent_dim)(hidden)
def sampling(args):
z_mean, z_log_var = args
return K.random_normal(
shape=K.shape(z_log_var), mean=0., stddev=noise_std) * K.exp(
.5 * z_log_var) + z_mean
z = Lambda(sampling)([z_mean, z_log_var])
encoding_shape = K.int_shape(z)
if verbose:
print("Encoding shape is", encoding_shape, "(", latent_dim,
"dimensions )")
# this model maps an input to its encoded representation
encoder = Model(input_img, z)
# next, declare the auto_encoding output side
n = 0
n += 1
ae = Dense(intermediate_dim, activation='selu')(z)
n += 1
ae = Reshape(conv_output_shape[1:])(ae)
n += 1
ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae)
n += 1
ae = LeakyReLU(alpha=0.05)(ae)
n += 1
ae = BatchNormalization()(ae)
n += 1
ae = UpSampling2D((2, 2))(ae)
n += 1
ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae)
n += 1
ae = LeakyReLU(alpha=0.05)(ae)
n += 1
ae = BatchNormalization()(ae)
n += 1
ae = UpSampling2D((2, 2))(ae)
n += 1
ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae)
n += 1
ae = LeakyReLU(alpha=0.05)(ae)
n += 1
ae = UpSampling2D((2, 2))(ae)
n += 1
decoded = Conv2D(1, (2, 2),
padding='same')(ae) # activation='sigmoid',
if verbose:
print("Decoder output shape is", decoded.get_shape())
# this is a pipe from the input image to the reconstructed output
autoencoder = Model(input_img, decoded)
# use right side of architecture encoded input to construct an image
encoded_input = Input(shape=encoding_shape[1:])
deco = encoded_input
for l in range(-n, 0):
deco = autoencoder.layers[l](deco)
decoder = Model(encoded_input, deco)
# and then, the classifier
n = 0
n += 1
cl = Dense(filters, activation='selu')(z)
n += 1
cl = AlphaDropout(0.1)(cl)
n += 1
cl = Dense(filters, activation='selu')(cl)
n += 1
cl = AlphaDropout(0.1)(cl)
n += 1
cl = Dense(filters, activation='selu')(cl)
n += 1
classified = Dense(num_categories, activation='softmax')(cl)
if verbose:
print("Classifier output shape is", classified.get_shape())
# provide classification on images
imageclassifier = Model(input_img, classified)
# imageclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
# and classifications on encoded representations
encoded_input = Input(shape=(encoding_shape[1:]))
fc = encoded_input
for l in range(-n, 0):
fc = imageclassifier.layers[l](fc)
featureclassifier = Model(encoded_input, fc)
# featureclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])
# Some KL loss
def vae_objective(x, x_decoded):
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
base_loss = tf.reduce_sum(metrics.binary_crossentropy(
x, x_decoded),
axis=[-1, -2])
return base_loss + kl_loss
# Finally, compile the full model (1 input, 2 outputs)
classycoder = Model(inputs=[input_img], outputs=[decoded, classified])
# = make_parallel(classycoder, 2) # Todo?
optimizer = Nadam(lr=0.0002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004,
clipnorm=0.618)
# = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.0)
classycoder.compile(optimizer=optimizer,
loss=[vae_objective, 'categorical_crossentropy'],
loss_weights=[0.1, 0.9],
metrics=['acc'])
# API mapping
self.encoding_dims = latent_dim
self.encoding_shape = encoding_shape
# Recieve a image and encode it into its latent space representation
if parallel_mode: self.encoder = make_parallel(encoder, config.GPUs)
else: self.encoder = encoder
# reconstructs an image from its encoded representation
if parallel_mode: self.decoder = make_parallel(decoder, config.GPUs)
else: self.decoder = decoder
# direct pipe from input_image to its classification
if parallel_mode:
self.imageclassifier = make_parallel(imageclassifier, config.GPUs)
else:
self.imageclassifier = imageclassifier
# direct pipe from encoded representation to classification
if parallel_mode:
self.featureclassifier = make_parallel(featureclassifier,
config.GPUs)
else:
self.featureclassifier = featureclassifier
# multiple output model, train this for the rest to work. # todo: make_parallel
self.classycoder = classycoder
def __init__(self,
lr=0.002,
lat_input_shape=(64, ),
screen_input_shape=(
64,
64,
),
structured_input_shape=(2, ),
verbose=False):
"""
https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models
https://keras.io/getting-started/functional-api-guide/#shared-layers
https://blog.keras.io/building-autoencoders-in-keras.html
"""
self.lr = lr
structured_input = Input(shape=structured_input_shape,
name='structured_input')
lat_input = Input(shape=lat_input_shape, name='latent_vector_input')
screen_input = Input(shape=screen_input_shape, name='screen_input')
eng_state = [structured_input, lat_input, screen_input]
if verbose:
print("Network structured input shape is",
structured_input.get_shape())
print("Network screen input shape is", screen_input.get_shape())
print("Network latent input shape is", lat_input.get_shape())
# Broadcast the structured information along
# channel dimension.
x = RepeatVector(32)(structured_input)
structured_output = Reshape((64, ))(x)
lat_output = lat_input
# newShape = tuple(list(screen_input_shape)+list(structured_input_shape))
# x = Reshape(newShape)(x)
# newShape = tuple(list(screen_input_shape)+[1])
# y = Reshape(newShape)(screen_input)
# x = concatenate([y, x])
newShape = tuple(list(screen_input_shape) + [1])
x = Reshape(newShape)(screen_input)
x = Conv2D(16, (5, 5))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(2)(x)
x = Conv2D(32, (3, 3))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(32, (3, 3))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(2)(x)
x = Flatten()(x)
screen_output = Dense(64)(x)
print("screen", screen_output.shape)
print("struct", structured_output.shape)
print("latent", lat_output.shape)
x = concatenate([screen_output, lat_input, structured_output])
x = Dense(256)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
state_output = Dense(128)(x)
state_network = Model(inputs=eng_state,
outputs=[state_output],
name='stateNetwork')
# Create the two state encoding legs
structured_input_a = Input(shape=structured_input_shape,
name='structured_input_a')
lat_input_a = Input(shape=lat_input_shape,
name='latent_vector_input_a')
screen_input_a = Input(shape=screen_input_shape, name='screen_input_a')
eng_state_a = [structured_input_a, lat_input_a, screen_input_a]
structured_input_b = Input(shape=structured_input_shape,
name='structured_input_b')
lat_input_b = Input(shape=lat_input_shape,
name='latent_vector_input_b')
screen_input_b = Input(shape=screen_input_shape, name='screen_input_b')
eng_state_b = [structured_input_b, lat_input_b, screen_input_b]
enc_state_a = state_network(eng_state_a)
enc_state_b = state_network(eng_state_b)
# Create the probability network
prob_input = concatenate([enc_state_a, enc_state_b])
x = Dense(256)(prob_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dense(128)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = BatchNormalization()(x)
x = Dense(64)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
prob_output = Dense(2, activation='softmax', name="probOutput")(x)
self.probabilityNetwork = Model(inputs=eng_state_a + eng_state_b,
outputs=[prob_output])
