def __init__(self, input_shape=(778, 576, 1), num_categories=5, parallel_mode=False, verbose=False): """ https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models https://keras.io/getting-started/functional-api-guide/#shared-layers https://blog.keras.io/building-autoencoders-in-keras.html https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py """ self.num_categories = num_categories filters = 128 img_rows, img_cols, img_chns = input_shape input_img = Input(shape=input_shape, name="main_input") if verbose: print("Network input shape is", input_img.get_shape()) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(input_img) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.05)(x) self.framer = Model(input_img, x) optimizer = Nadam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004, clipnorm=0.618) self.framer.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['acc'])
def super_resolution(input_tensor, n_feature_layers=2, n_projection=1, feature_filters=[128, 32], projection_filters=32, k_size=[3, 1], s=2): ''' input_tensor: The input tensor required to complete the model n_features_layer: The number of initial feature extraction layers that needs to be fixed before starting the up and down projection blocks n_projection: Number of Up and down projection blocks feature_filters: Number of filters in each feature extraction layer k_size: The kernel size to be used in feature extraction layer s: The scaling factor of the super resolution network Returns ------- A Model object of the super resolution network ''' if not K_B.is_keras_tensor(input_tensor): input_tensor = Input(shape=input_tensor.get_shape()[1:], tensor=input_tensor) x = Conv2D(feature_filters[0], k_size[0], padding="SAME")(input_tensor) if (len(k_size) != len(feature_filters)): raise ValueError( "Number of Layers for feature extraction must be equal to the number of filters sets given." ) for i in range(len(k_size) - 1): x = Conv2D(feature_filters[i + 1], k_size[i + 1], padding="same")(x) for i in range(n_projection): x = up_projection(x, projection_filters, s, i + 1) x = down_projection(x, projection_filters, s, i + 1) x = up_projection(x, projection_filters, s, n_projection + 1) x = Conv2D(3, 3, padding='same')(x) return Model(inputs=input_tensor, outputs=x)
def __init__(self, input_shape=(64, 64, 1), num_categories=5, parallel_mode=False, verbose=False): """ https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models https://keras.io/getting-started/functional-api-guide/#shared-layers https://blog.keras.io/building-autoencoders-in-keras.html https://github.com/fchollet/keras/blob/master/examples/variational_autoencoder_deconv.py """ self.num_categories = num_categories # number of filters # convolution kernel size filters = 128 latent_dim = 64 epsilon_std = 1.0 noise_std = .01 img_rows, img_cols, img_chns = input_shape input_img = Input(shape=input_shape, name="main_input") if verbose: print("Network input shape is", input_img.get_shape()) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(input_img) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(filters, (3, 3), padding='same', activity_regularizer=l2(10e-8))(x) x = LeakyReLU(alpha=0.05)(x) x = BatchNormalization()(x) x = MaxPooling2D((2, 2), padding='same', name="encoded")(x) conv_output_shape = K.int_shape(x) intermediate_dim = conv_output_shape[1] * conv_output_shape[ 2] * conv_output_shape[3] if verbose: print("Convolution output shape is", conv_output_shape) x = Flatten()(x) hidden = Dense(intermediate_dim, activation='selu')(x) z_mean = Dense(latent_dim)(hidden) z_log_var = Dense(latent_dim)(hidden) def sampling(args): z_mean, z_log_var = args return K.random_normal( shape=K.shape(z_log_var), mean=0., stddev=noise_std) * K.exp( .5 * z_log_var) + z_mean z = Lambda(sampling)([z_mean, z_log_var]) encoding_shape = K.int_shape(z) if verbose: print("Encoding shape is", encoding_shape, "(", latent_dim, "dimensions )") # this model maps an input to its encoded representation encoder = Model(input_img, z) # next, declare the auto_encoding output side n = 0 n += 1 ae = Dense(intermediate_dim, activation='selu')(z) n += 1 ae = Reshape(conv_output_shape[1:])(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = BatchNormalization()(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = BatchNormalization()(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 ae = Conv2DTranspose(filters, (3, 3), padding='same')(ae) n += 1 ae = LeakyReLU(alpha=0.05)(ae) n += 1 ae = UpSampling2D((2, 2))(ae) n += 1 decoded = Conv2D(1, (2, 2), padding='same')(ae) # activation='sigmoid', if verbose: print("Decoder output shape is", decoded.get_shape()) # this is a pipe from the input image to the reconstructed output autoencoder = Model(input_img, decoded) # use right side of architecture encoded input to construct an image encoded_input = Input(shape=encoding_shape[1:]) deco = encoded_input for l in range(-n, 0): deco = autoencoder.layers[l](deco) decoder = Model(encoded_input, deco) # and then, the classifier n = 0 n += 1 cl = Dense(filters, activation='selu')(z) n += 1 cl = AlphaDropout(0.1)(cl) n += 1 cl = Dense(filters, activation='selu')(cl) n += 1 cl = AlphaDropout(0.1)(cl) n += 1 cl = Dense(filters, activation='selu')(cl) n += 1 classified = Dense(num_categories, activation='softmax')(cl) if verbose: print("Classifier output shape is", classified.get_shape()) # provide classification on images imageclassifier = Model(input_img, classified) # imageclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) # and classifications on encoded representations encoded_input = Input(shape=(encoding_shape[1:])) fc = encoded_input for l in range(-n, 0): fc = imageclassifier.layers[l](fc) featureclassifier = Model(encoded_input, fc) # featureclassifier.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) # Some KL loss def vae_objective(x, x_decoded): kl_loss = -0.5 * K.sum( 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) base_loss = tf.reduce_sum(metrics.binary_crossentropy( x, x_decoded), axis=[-1, -2]) return base_loss + kl_loss # Finally, compile the full model (1 input, 2 outputs) classycoder = Model(inputs=[input_img], outputs=[decoded, classified]) # = make_parallel(classycoder, 2) # Todo? optimizer = Nadam(lr=0.0002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004, clipnorm=0.618) # = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.0) classycoder.compile(optimizer=optimizer, loss=[vae_objective, 'categorical_crossentropy'], loss_weights=[0.1, 0.9], metrics=['acc']) # API mapping self.encoding_dims = latent_dim self.encoding_shape = encoding_shape # Recieve a image and encode it into its latent space representation if parallel_mode: self.encoder = make_parallel(encoder, config.GPUs) else: self.encoder = encoder # reconstructs an image from its encoded representation if parallel_mode: self.decoder = make_parallel(decoder, config.GPUs) else: self.decoder = decoder # direct pipe from input_image to its classification if parallel_mode: self.imageclassifier = make_parallel(imageclassifier, config.GPUs) else: self.imageclassifier = imageclassifier # direct pipe from encoded representation to classification if parallel_mode: self.featureclassifier = make_parallel(featureclassifier, config.GPUs) else: self.featureclassifier = featureclassifier # multiple output model, train this for the rest to work. # todo: make_parallel self.classycoder = classycoder
def __init__(self, lr=0.002, lat_input_shape=(64, ), screen_input_shape=( 64, 64, ), structured_input_shape=(2, ), verbose=False): """ https://keras.io/getting-started/functional-api-guide/#multi-input-and-multi-output-models https://keras.io/getting-started/functional-api-guide/#shared-layers https://blog.keras.io/building-autoencoders-in-keras.html """ self.lr = lr structured_input = Input(shape=structured_input_shape, name='structured_input') lat_input = Input(shape=lat_input_shape, name='latent_vector_input') screen_input = Input(shape=screen_input_shape, name='screen_input') eng_state = [structured_input, lat_input, screen_input] if verbose: print("Network structured input shape is", structured_input.get_shape()) print("Network screen input shape is", screen_input.get_shape()) print("Network latent input shape is", lat_input.get_shape()) # Broadcast the structured information along # channel dimension. x = RepeatVector(32)(structured_input) structured_output = Reshape((64, ))(x) lat_output = lat_input # newShape = tuple(list(screen_input_shape)+list(structured_input_shape)) # x = Reshape(newShape)(x) # newShape = tuple(list(screen_input_shape)+[1]) # y = Reshape(newShape)(screen_input) # x = concatenate([y, x]) newShape = tuple(list(screen_input_shape) + [1]) x = Reshape(newShape)(screen_input) x = Conv2D(16, (5, 5))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(32, (3, 3))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(2)(x) x = Conv2D(32, (3, 3))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(32, (3, 3))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(2)(x) x = Flatten()(x) screen_output = Dense(64)(x) print("screen", screen_output.shape) print("struct", structured_output.shape) print("latent", lat_output.shape) x = concatenate([screen_output, lat_input, structured_output]) x = Dense(256)(x) x = BatchNormalization()(x) x = Activation('relu')(x) state_output = Dense(128)(x) state_network = Model(inputs=eng_state, outputs=[state_output], name='stateNetwork') # Create the two state encoding legs structured_input_a = Input(shape=structured_input_shape, name='structured_input_a') lat_input_a = Input(shape=lat_input_shape, name='latent_vector_input_a') screen_input_a = Input(shape=screen_input_shape, name='screen_input_a') eng_state_a = [structured_input_a, lat_input_a, screen_input_a] structured_input_b = Input(shape=structured_input_shape, name='structured_input_b') lat_input_b = Input(shape=lat_input_shape, name='latent_vector_input_b') screen_input_b = Input(shape=screen_input_shape, name='screen_input_b') eng_state_b = [structured_input_b, lat_input_b, screen_input_b] enc_state_a = state_network(eng_state_a) enc_state_b = state_network(eng_state_b) # Create the probability network prob_input = concatenate([enc_state_a, enc_state_b]) x = Dense(256)(prob_input) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dense(128)(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = Dense(64)(x) x = BatchNormalization()(x) x = Activation('relu')(x) prob_output = Dense(2, activation='softmax', name="probOutput")(x) self.probabilityNetwork = Model(inputs=eng_state_a + eng_state_b, outputs=[prob_output])