from keras.optimizers import SGD,Adam from keras.losses import binary_crossentropy from keras.callbacks import EarlyStopping, TensorBoard, LearningRateScheduler, ModelCheckpoint import numpy.random as rng import para import random import copy import time import numpy as np import tensorflow as tf import keras import generator from keras.initializers import RandomNormal convnet = Sequential() convnet.add(Conv2D(32,(5,5),activation='relu',input_shape=para.input_shape3, kernel_initializer=RandomNormal(0,1e-2) ,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2))) convnet.add(MaxPooling2D()) convnet.add(SpatialDropout2D(0.12)) convnet.add(Conv2D(64,(3,3),activation='relu', kernel_initializer=RandomNormal(0,1e-2) ,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2))) convnet.add(MaxPooling2D()) convnet.add(SpatialDropout2D(0.12)) convnet.add(Conv2D(128,(2,2),activation='relu', kernel_initializer=RandomNormal(0,1e-2) ,kernel_regularizer=l2(2e-4),bias_initializer=RandomNormal(0.5,1e-2))) convnet.add(Flatten())
import numpy as np import keras from keras.initializers import RandomNormal from keras.layers import Dense, LeakyReLU from keras.regularizers import l2 import chess_environment.chessboard as cb from dqn_tools.memory import SimpleMemory from dqn_tools.trainers import DQNTrainer, load_trainer from training_tools import DQNChessRecord seed = 12345 np.random.seed(seed) # temporary simple model for testing base concept weight_decay = l2(1e-2) weight_initializer = RandomNormal(mean=0., stddev=0.02, seed=seed) model = keras.Sequential([ Dense(150, input_shape=(384, ), kernel_initializer=weight_initializer, bias_initializer=weight_initializer, kernel_regularizer=weight_decay, bias_regularizer=weight_decay), LeakyReLU(alpha=0.3), Dense(300, kernel_initializer=weight_initializer, bias_initializer=weight_initializer, kernel_regularizer=weight_decay, bias_regularizer=weight_decay), LeakyReLU(alpha=0.3), Dense(1,
def __conv_init(a): print("conv_init", a) k = RandomNormal(0, 0.02)(a) # for convolution kernel k.conv_weight = True return k
def mod_testing2(inp: Union[Tensor, Layer], kernel_initializer: Initializer = RandomNormal(stddev=0.02)): m = conv_layer(inp, 64, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 64, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 256, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 256, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 512, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 512, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = GlobalAveragePooling2D()(m) m = Dropout(0.4)(m) m = Dense(1024, activation=None)(m) m = LeakyReLU(0.2)(m) return m
def build_patch_discriminator_(self, model_shape, filters=32, k_size=4, drop=False, rate=0.5, summary=False, model_file=None, name='gan_d_'): """ Create a Discriminator Model using hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: """ Create a Discriminator Model using hyperparameters values defined as follows """ init = RandomNormal(stddev=0.02) n_rows = model_shape[0] n_cols = model_shape[1] c_dims = model_shape[2] input_shape = (n_rows, n_cols, c_dims) input_layer = Input(shape=input_shape, name=name + 'input') d = self.Conv2D_Block(input_layer, filters, k_size=k_size, name=name + '1', bn=False) d = self.Conv2D_Block(d, 2 * filters, k_size=k_size, insnorm=True, name=name + '2') d = self.Conv2D_Block(d, 4 * filters, k_size=k_size, insnorm=True, name=name + '3') d = self.Conv2D_Block(d, 8 * filters, k_size=k_size, insnorm=True, strides=1, name=name + '4') if drop: d = Dropout(rate=0.5, name=name + '_dropout')(d, training=True) logits = Conv2D(1, k_size, strides=1, padding='same', kernel_initializer=init, name=name + 'logits')(d) out = Activation('sigmoid', name=name + 'sigmoid')(logits) model = Model(inputs=[input_layer], outputs=[out, logits], name='Discriminator_' + name[-3:]) if (summary): model.summary() return model
def build_resnet_generator_insnorm(self, model_shape, filters=32, k_size=3, last_act='tanh', n_residuals=9, summary=False, model_file=None, name='gan_g_'): """ Create a Generator Model with hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: init = RandomNormal(stddev=0.02) n_rows = model_shape[0] n_cols = model_shape[1] in_c_dims = model_shape[2] out_c_dims = model_shape[3] n_rows_e1, n_rows_e2, n_rows_e4, n_rows_e8 = n_rows // 1, n_rows // 2, n_rows // 4, n_rows // 8 rows_matching = np.equal( [2 * n_rows_e2, 2 * n_rows_e4, 2 * n_rows_e8], [n_rows_e1, n_rows_e2, n_rows_e4]) index_rows = np.where(np.logical_not(rows_matching))[0] n_cols_e1, n_cols_e2, n_cols_e4, n_cols_e8 = n_cols // 1, n_cols // 2, n_cols // 4, n_cols // 8 cols_matching = np.equal( [2 * n_cols_e2, 2 * n_cols_e4, 2 * n_cols_e8], [n_cols_e1, n_cols_e2, n_cols_e4]) index_cols = np.where(np.logical_not(cols_matching))[0] input_shape = (n_rows, n_cols, in_c_dims) input_layer = Input(shape=input_shape, name=name + '_input') x = self.Conv2D_Block(input_layer, filters, k_size=7, strides=1, activation='relu', bn=False, name=name + 'e1') # rows, cols x = self.Conv2D_Block(x, 2 * filters, k_size=k_size, insnorm=True, activation='relu', name=name + 'e2') # rows/2, cols/2 x = self.Conv2D_Block(x, 4 * filters, k_size=k_size, insnorm=True, activation='relu', name=name + 'e3') # rows/4, cols/4 for i in range(n_residuals): x = self.residual_block(x, n_kernels=4 * filters, k_size=k_size, activation='relu', insnorm=True, add=False, name=name + str(i + 1) + '_') x = self.Conv2DTranspose_Block(x, 2 * filters, k_size=k_size, insnorm=True, name=name + 'd1') # rows/2, cols/2 x = self.Conv2DTranspose_Block(x, 1 * filters, k_size=k_size, insnorm=True, name=name + 'd2') # rows, cols x = Conv2DTranspose(out_c_dims, 7, strides=1, padding='same', kernel_initializer=init, name=name + 'd_out')(x) # rows, cols x = InstanceNormalization(axis=-1, name=name + 'ins_norm')(x) output = Activation(last_act, name=name + last_act)(x) model = Model(inputs=[input_layer], outputs=[output], name='Generator' + name[-3:]) if (summary): model.summary() return model
def vdcnn_model(embedding_size=97, seq_maxlen=512, n_quantized_characters=97): """Very Deep CNN model, uses encoded character level data https://arxiv.org/pdf/1606.01781.pdf""" input_text = Input(shape=(seq_maxlen, )) x = Embedding(input_dim=n_quantized_characters, output_dim=embedding_size, input_length=seq_maxlen)(input_text) #x = _convolutional_block(64)(x) x = Conv1D(filters=64, kernel_size=3, strides=2, padding="same")(x) #filters = [64, 128, 256, 512] filters = [64, 128, 256, 512, 1024] for n_filt in filters: ### convolutional block x = Conv1D(n_filt, kernel_size=3, strides=1, padding='same', activation='linear', kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x) x = BatchNormalization()(x) x = PReLU()(x) #PReLU x = Conv1D(n_filt, kernel_size=3, strides=1, padding='same', activation='linear', kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x) x = BatchNormalization()(x) x = PReLU()(x) ### #if n_filt != filters[-1]: # print("adding", n_filt, filters[-1]) x = MaxPooling1D(pool_size=3, strides=2, padding='same')(x) # k-max pooling (Finds values and indices of the k largest entries for the last dimension) top_k = 8 def _top_k(x): x = tf.transpose(x, [0, 2, 1]) k_max = tf.nn.top_k(x, k=top_k) return tf.reshape(k_max[0], (-1, filters[-1] * top_k)) k_max = Lambda(_top_k, output_shape=(filters[-1] * top_k, ))(x) #x = GlobalMaxPool1D()(x) x = Dropout(0.2)(Dense(128)(k_max)) x = PReLU()(x) x = Dropout(0.2)(Dense(128)(x)) x = PReLU()(x) y_pred = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_text, outputs=y_pred) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', auc_roc]) return model
def build_generator2D_(self, model_shape, filters=32, k_size=4, z_size=500, summary=False, model_file=None, name='gan_g_'): """ Create a Generator Model with hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: n_rows = model_shape[0] n_cols = model_shape[1] c_dims = model_shape[2] input_shape = (z_size, ) if n_rows % 8 != 0: height = n_rows // 8 + 1 else: height = n_rows // 8 if n_cols % 8 != 0: width = n_cols // 8 + 1 else: width = n_cols // 8 num_init_neurons = 8 * filters reshape_size = (height, width, num_init_neurons) # 8*height, 4*height, 2*height, height = n_rows, n_rows//2, n_rows//4, n_rows//8 rows_matching = np.equal([2 * height, 4 * height, 8 * height], [n_rows // 4, n_rows // 2, n_rows]) index_rows = np.where(np.logical_not(rows_matching))[0] if len(index_rows) > 0: index_rows = index_rows[0] # print(index_rows) # 8*width, 4*width, 2*width, width = n_cols//1, n_cols//2, n_cols//4, n_cols//8 cols_matching = np.equal([2 * width, 4 * width, 8 * width], [n_cols // 4, n_cols // 2, n_cols]) index_cols = np.where(np.logical_not(cols_matching))[0] if len(index_cols) > 0: index_cols = index_cols[0] # print(index_cols) input_layer = Input(shape=input_shape, name=name + 'input') g = Dense(width * height * num_init_neurons, kernel_initializer=RandomNormal(stddev=0.02), name=name + 'dense')(input_layer) g = Reshape(reshape_size, name=name + 'reshape')(g) g = BatchNormalization(momentum=0.8, name=name + 'bn_dense')(g, training=True) g = Activation(activation='relu', name=name + 'relu')(g) g = self.Conv2DTranspose_Block(g, 4 * filters, name=name + '1') if index_rows == 0 or index_cols == 0: g = BilinearUpsampling(output_size=(n_rows // 4, n_cols // 4), name=name + 'bilinear')(g) g = self.Conv2DTranspose_Block(g, 2 * filters, k_size=k_size, name=name + '2') if index_rows == 1 or index_cols == 1: g = BilinearUpsampling(output_size=(n_rows // 2, n_cols // 2), name=name + 'bilinear')(g) g = self.Conv2DTranspose_Block(g, 1 * filters, k_size=k_size, name=name + '3') if index_rows == 2 or index_cols == 2: g = BilinearUpsampling(output_size=(n_rows, n_cols), name=name + 'bilinear')(g) g = self.Conv2DTranspose_Block(g, c_dims, strides=1, activation='tanh', k_size=k_size, name=name + '4', bn=False) model = Model(inputs=[input_layer], outputs=[g], name='Generator') if (summary): model.summary() return model
def convolutional(config, container): if not config.module_name: raise ValueError('Missing module name in section') config.layer_name = ( config.layer_name if config.layer_name else str(len(container.all_layers) - 1) ) config.size = int(config.size) if container.frames > 1: logging.info(f"frames: {container.frames}") prev_layer_shape = K.int_shape(container.all_layers[-1]) input_channels = prev_layer_shape[-1] image_size = prev_layer_shape[-3], prev_layer_shape[-2] num_convs = int(container.frames / config.tstride) if num_convs > 1: logging.info(f"num_convs: {num_convs}") inputs_needed = (config.tstride * (num_convs - 1)) + config.tsize # inputs_needed = frames + tsize - 1 if inputs_needed > 1: logging.info(f"inputs_needed: {inputs_needed}") old_frames_to_read = inputs_needed - container.frames if old_frames_to_read < 0: logging.info("negative number of old frames!!!!!!!!!") if old_frames_to_read: logging.info(f"history frames needed: {old_frames_to_read}") new_frames_to_save = min(container.frames, old_frames_to_read) if new_frames_to_save: logging.info(f"new frames to save: {new_frames_to_save}") # attach output ports to inputs we will need next pass if config.no_output is False: for f in range(new_frames_to_save): container.out_index.append(len(container.all_layers) - container.frames + f) container.out_names.append(config.module_name + "_save_" + str(f)) # attach output ports to unsaved inputs if we need to share inputs to a slave network if config.share is True: for f in range(new_frames_to_save, container.frames): container.out_index.append(len(container.all_layers) - container.frames + f) container.out_names.append(config.module_name + "_share_" + str(f)) # create input ports for required old frames for f in range(old_frames_to_read): xx = config.module_name + "_history_" + str(f) container.in_names.append(xx) if config.image_input: container.image_inputs.append(xx) container.all_layers.append( Input(shape=(image_size[0], image_size[1], input_channels), name=xx) ) container.in_index.append(len(container.all_layers) - 1) # create input ports for merged-in frames if config.merge_in > 0: input_channels = input_channels + config.merge_in for f in range(inputs_needed): xx = config.module_name + "_merge_in_" + str(f) container.in_names.append(xx) container.all_layers.append( Input(shape=(image_size[0], image_size[1], config.merge_in), name=xx) ) logging.info(f"merge_in input at: {len(container.all_layers) - 1}") container.in_index.append(len(container.all_layers) - 1) padding = "same" if config.pad == 1 and config.stride == 1 else "valid" # extract parameter for this module from Pytorch checkpoint file conv_weights_pt = np.random.rand( input_channels, config.filters, config.tsize, config.size, config.size ) conv_bias = [0] if config.module_name + ".weight" in container.weights: conv_weights_pt = container.weights[config.module_name + ".weight"] shape = container.weights[config.module_name + ".weight"].shape logging.info(f"weight: {config.module_name}.weight {shape}") # convert to tsize list of 2d conv weight matrices, transposed for Keras w_list = [] if len(conv_weights_pt.shape) == 5: # check if this is a 3D conv being unfolded for t in range(config.tsize): w_list.append( np.transpose( conv_weights_pt[:, :, config.tsize - 1 - t, :, :], [2, 3, 1, 0] ) ) else: # this is simply a single 2D conv w_list.append(np.transpose(conv_weights_pt[:, :, :, :], [2, 3, 1, 0])) # concatenate along the in_dim axis the tsize matrices conv_weights = np.concatenate(w_list, axis=2) if not config.batch_normalize: conv_bias = container.weights[config.module_name + ".bias"] else: logging.info(f"cannot find weight: {config.module_name}.weight") container.fake_weights = True conv_weights = np.random.rand( config.size, config.size, config.tsize * input_channels, config.filters ) conv_bias = np.zeros(config.filters) if config.batch_normalize: bn_bias = container.weights[config.module_name + ".batchnorm.bias"] bn_weight = container.weights[config.module_name + ".batchnorm.weight"] bn_running_var = container.weights[ config.module_name + ".batchnorm.running_var" ] bn_running_mean = container.weights[ config.module_name + ".batchnorm.running_mean" ] bn_weight_list = [ bn_weight, # scale gamma bn_bias, # shift beta bn_running_mean, # running mean bn_running_var, # running var ] expected_weights_shape = ( config.size, config.size, config.tsize * input_channels, config.filters, ) logging.info( f"weight shape, expected : {expected_weights_shape} " f"checkpoint: {conv_weights_pt.shape} " f"created: {conv_weights.shape} " ) if conv_weights.shape != expected_weights_shape: logging.info("weight matrix shape is wrong, making a fake one") container.fake_weights = True conv_weights = np.random.rand( config.size, config.size, config.tsize * input_channels, config.filters ) conv_bias = np.zeros(config.filters) conv_weights = ( [conv_weights] if config.batch_normalize else [conv_weights, conv_bias] ) inputs = [] outputs = [] for f in range(inputs_needed): inputs.append( container.all_layers[len(container.all_layers) - inputs_needed + f] ) if config.merge_in > 0: inputs.append( container.all_layers[ len(container.all_layers) - (2 * inputs_needed) + f ] ) # Create Conv3d from Conv2D layers if config.stride > 1: for f in range(len(inputs)): inputs[f] = ZeroPadding2D(((1, 0), (1, 0)))(inputs[f]) for f in range(int(container.frames / config.tstride)): layers = [] if config.tsize > 1: for t in range(config.tsize): # offset is constant with f, except if tstride, # then steps by extra step every time through if config.merge_in == 0: layers.append(inputs[t + (f * (config.tstride))]) else: layers.append(inputs[2 * (t + (f * (config.tstride)))]) layers.append(inputs[2 * (t + (f * (config.tstride))) + 1]) cat_layer = Concatenate()(layers) else: if config.merge_in == 0: cat_layer = inputs[f * (config.tstride)] else: layers.append(inputs[2 * f * (config.tstride)]) layers.append(inputs[2 * f * (config.tstride) + 1]) cat_layer = Concatenate()(layers) outputs.append( ( Conv2D( config.filters, (config.size, config.size), strides=(config.stride, config.stride), kernel_regularizer=l2(5e-4), use_bias=not config.batch_normalize, weights=conv_weights, activation=None, padding=padding, ) )(cat_layer) ) if config.batch_normalize: for f in range(int(container.frames / config.tstride)): outputs[f] = BatchNormalization(weights=bn_weight_list)(outputs[f]) if config.activation == "relu6": for f in range(int(container.frames / config.tstride)): outputs[f] = ReLU(max_value=6)(outputs[f]) elif config.activation == "leaky": for f in range(int(container.frames / config.tstride)): if not container.conversion_parameters["use_prelu"]: outputs[f] = LeakyReLU(alpha=0.1)(outputs[f]) else: outputs[f] = PReLU( alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None), shared_axes=[1, 2], )(outputs[f]) for f in range(int(container.frames / config.tstride)): container.all_layers.append(outputs[f]) frames = int(container.frames / config.tstride) if frames == 0: raise ValueError("tried to time stride single frame") container.layer_names[config.layer_name] = len(container.all_layers) - 1
def build_model(): model = Sequential() model.add( Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.01), input_shape=x_train.shape[1:])) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(160, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(96, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')) model.add(Dropout(dropout)) model.add( Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')) model.add(Dropout(dropout)) model.add( Conv2D(192, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add( Conv2D(10, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(GlobalAveragePooling2D()) model.add(Activation('softmax')) sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model
def invResidual(config, container): if not config.module_name: raise ValueError('Missing module name in section') config.layer_name = ( config.layer_name if config.layer_name else str(len(container.all_layers) - 1) ) logging.info(f"frames: {container.frames}") s = 0 if config.shift: logging.info("3D conv block") tsize = 3 else: logging.info("2D conv block") tsize = 1 config.size = int(config.size) prev_layer = container.all_layers[-1] prev_layer_shape = K.int_shape(prev_layer) input_channels = prev_layer_shape[-1] x_channels = input_channels * config.xratio image_size = prev_layer_shape[-3], prev_layer_shape[-2] logging.info(f"input image size: {image_size}") num_convs = int(container.frames / config.tstride) inputs_needed = (config.tstride * (num_convs - 1)) + tsize # inputs_needed = frames + tsize - 1 if inputs_needed > 1: logging.info(f"inputs_needed: {inputs_needed}") old_frames_to_read = inputs_needed - container.frames new_frames_to_save = min(container.frames, old_frames_to_read) logging.info( f"num_convs: {num_convs}, inputs_needed: {inputs_needed}, history frames needed: {old_frames_to_read}," f"frames to save: {new_frames_to_save}, tstride: {config.tstride}" ) # create (optional) expansion pointwise convolution layer input_indexes = [] for i in range(num_convs): input_indexes.append( len(container.all_layers) - container.frames + (i * config.tstride) ) if config.xratio != 1: logging.info("---------- Insert channel multiplier pointwise conv -------------") # attach output ports to inputs we will need next pass if tsize>1 for f in range(new_frames_to_save): container.out_index.append(len(container.all_layers) - container.frames + f) container.out_names.append(config.module_name + "_save_" + str(f)) # create input ports for required old frames if tsize>1 for f in range(old_frames_to_read): h_name = config.module_name + "_history_" + str(f) container.all_layers.append( Input(shape=(image_size[0], image_size[1], input_channels), name=h_name) ) container.in_names.append(h_name) container.in_index.append(len(container.all_layers) - 1) # get weights n = config.module_name + ".conv." + str(s) + ".0." if n + "weight" in container.weights: weights_pt = container.weights[n + "weight"] logging.info(f"checkpoint: {weights_pt.shape}") weights_k = np.transpose(weights_pt, [2, 3, 1, 0]) bias = container.weights[n + "bias"] else: logging.info(f"missing weight {n}weight") weights_k = np.random.rand(1, 1, tsize * input_channels, x_channels) bias = np.zeros(x_channels) container.fake_weights = True expected_weights_shape = (1, 1, tsize * input_channels, x_channels) logging.info( f"weight shape, expected : {expected_weights_shape} transposed: {weights_k.shape}" ) if weights_k.shape != expected_weights_shape: logging.info("weight matrix shape is wrong, making a fake one") weights_k = np.random.rand(1, 1, tsize * input_channels, x_channels) bias = np.zeros(x_channels) container.fake_weights = True weights = [weights_k, bias] inputs = [] outputs = [] for f in range(inputs_needed): inputs.append( container.all_layers[len(container.all_layers) - inputs_needed + f] ) if config.merge_in > 0: inputs.append( container.all_layers[ len(container.all_layers) - (2 * inputs_needed) + f ] ) for f in range(int(container.frames / config.tstride)): layers = [] if tsize > 1: for t in range(tsize): # offset is constant with f, except if tstride, # then steps by extra step every time through layers.append(inputs[(tsize - t - 1) + (f * (config.tstride))]) cat_layer = Concatenate()(layers) else: cat_layer = inputs[f * (config.tstride)] outputs.append( ( Conv2D( x_channels, (1, 1), use_bias=not config.batch_normalize, weights=weights, activation=None, padding="same", ) )(cat_layer) ) logging.info( f"parallel convs: {int(container.frames / config.tstride)} : {K.int_shape(cat_layer)}" ) if config.activation == "leaky": for f in range(int(container.frames / config.tstride)): if not container.conversion_parameters["use_prelu"]: outputs[f] = LeakyReLU(alpha=0.1)(outputs[f]) else: outputs[f] = PReLU( alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None), shared_axes=[1, 2], )(outputs[f]) elif config.activation == "relu6": for f in range(int(container.frames / config.tstride)): outputs[f] = ReLU(max_value=6)(outputs[f]) for f in range(int(container.frames / config.tstride)): container.all_layers.append(outputs[f]) s += 1 container.frames = int(container.frames / config.tstride) else: logging.info("Skipping channel multiplier pointwise conv, no expansion") # create groupwise convolution # get weights logging.info("---------- Depthwise conv -------------") n = config.module_name + ".conv." + str(s) + ".0." logging.info(f"module name base: {n}") if n + "weight" in container.weights: weights_pt = container.weights[n + "weight"] logging.info(f"checkpoint: {weights_pt.shape}") weights_k = np.transpose(weights_pt, [2, 3, 0, 1]) bias = container.weights[n + "bias"] else: logging.info(f"missing weight {n}weight") weights_k = np.random.rand(config.size, config.size, x_channels, 1) bias = np.zeros(x_channels) container.fake_weights = True expected_weights_shape = (config.size, config.size, x_channels, 1) logging.info(f"weight shape, expected : {expected_weights_shape} transposed: {weights_k.shape}") if weights_k.shape != expected_weights_shape: logging.error("weight matrix shape is wrong, making a fake one") container.fake_weights = True weights_k = np.random.rand(config.size, config.size, x_channels, 1) bias = np.zeros(x_channels) weights = [weights_k, bias] inputs = [] outputs = [] padding = "same" if config.pad == 1 and config.stride == 1 else "valid" for f in range(container.frames): inputs.append( container.all_layers[len(container.all_layers) - container.frames + f] ) if config.stride > 1: for f in range(len(inputs)): if config.size == 3: # originally for all sizes inputs[f] = ZeroPadding2D( ((config.size - config.stride, 0), (config.size - config.stride, 0)) )(inputs[f]) elif config.size == 5: # I found this works... inputs[f] = ZeroPadding2D(((2, 2), (2, 2)))(inputs[f]) else: logging.info(f"I have no idea what to do for size {config.size}") exit() logging.info(f"parallel convs: {f} : {K.int_shape(inputs[0])}, padding: {padding}") for f in range(container.frames): outputs.append( ( DepthwiseConv2D( (config.size, config.size), strides=(config.stride, config.stride), use_bias=not config.batch_normalize, weights=weights, activation=None, padding=padding, ) )(inputs[f]) ) if config.activation == "leaky": for f in range(int(container.frames)): if not container.conversion_parameters["use_prelu"]: outputs[f] = LeakyReLU(alpha=0.1)(outputs[f]) else: outputs[f] = PReLU( alpha_initializer=RandomNormal(mean=0.1, stddev=0.0, seed=None), shared_axes=[1, 2], )(outputs[f]) elif config.activation == "relu6": for f in range(int(container.frames)): outputs[f] = ReLU(max_value=6)(outputs[f]) for f in range(int(container.frames)): container.all_layers.append(outputs[f]) s += 1 # create pointwise convolution # get weights logging.info("---------- Pointwise conv -------------") n = config.module_name + ".conv." + str(s) + "." logging.info(f"module name base: {n}") if n + "weight" in container.weights: weights_pt = container.weights[n + "weight"] logging.info(f"checkpoint: {weights_pt.shape}") weights_k = np.transpose(weights_pt, [2, 3, 1, 0]) bias = container.weights[n + "bias"] else: logging.error(f"missing weight {n}weight") container.fake_weights = True weights_k = np.random.rand(1, 1, x_channels, config.out_channels) bias = np.zeros(config.out_channels) expected_weights_shape = (1, 1, x_channels, config.out_channels) logging.info( f"weight shape, expected : {expected_weights_shape}" f"transposed: {weights_k.shape}" ) if weights_k.shape != expected_weights_shape: logging.error("weight matrix shape is wrong, making a fake one") container.fake_weights = True weights_k = np.random.rand(1, 1, x_channels, config.out_channels) bias = np.zeros(config.out_channels) weights = [weights_k, bias] logging.info(f"combined shape: {weights[0].shape} {weights[1].shape}") inputs = [] outputs = [] for f in range(container.frames): inputs.append( container.all_layers[len(container.all_layers) - container.frames + f] ) shape = K.int_shape(container.all_layers[len(container.all_layers) - container.frames]) logging.info(f"parallel convs: {f} : {shape}") for f in range(container.frames): conv_input = container.all_layers[ len(container.all_layers) - container.frames + f ] outputs.append( ( Conv2D( config.out_channels, (1, 1), use_bias=not config.batch_normalize, weights=weights, activation=None, padding="same", ) )(conv_input) ) if config.stride == 1 and input_channels == config.out_channels: for f in range(int(container.frames)): container.all_layers.append( Add()([container.all_layers[input_indexes[f]], outputs[f]]) ) else: for f in range(int(container.frames)): container.all_layers.append(outputs[f]) s += 1
def generator_network(img_shape, filters, name): def resblock(feature_in, filters, num): init = RandomNormal(stddev=0.02) temp = Conv2D(filters, (3, 3), strides=1, padding='SAME', name=('resblock_%d_CONV_1' % num), kernel_initializer=init)(feature_in) temp = BatchNormalization(axis=-1)(temp) temp = Activation('relu')(temp) temp = Conv2D(filters, (3, 3), strides=1, padding='SAME', name=('resblock_%d_CONV_2' % num), kernel_initializer=init)(temp) temp = BatchNormalization(axis=-1)(temp) temp = Activation('relu')(temp) return Add()([temp, feature_in]) init = RandomNormal(stddev=0.02) image = Input(img_shape) y = Lambda(lambda x: 2.0 * x - 1.0, output_shape=lambda x: x)(image) b1_in = Conv2D(filters, (9, 9), strides=1, padding='SAME', name='CONV_1', activation='relu', kernel_initializer=init)(y) b1_in = Activation('relu')(b1_in) # residual blocks b1_out = resblock(b1_in, filters, 1) b2_out = resblock(b1_out, filters, 2) b3_out = resblock(b2_out, filters, 3) b4_out = resblock(b3_out, filters, 4) # conv. layers after residual blocks temp = Conv2D(filters, (3, 3), strides=1, padding='SAME', name='CONV_2', kernel_initializer=init)(b4_out) #temp = BatchNormalization(axis=-1)(temp) temp = Activation('relu')(temp) temp = Conv2D(filters, (3, 3), strides=1, padding='SAME', name='CONV_3', kernel_initializer=init)(temp) #temp = BatchNormalization(axis=-1)(temp) temp = Activation('relu')(temp) temp = Conv2D(filters, (3, 3), strides=1, padding='SAME', name='CONV_4', kernel_initializer=init)(temp) #temp = BatchNormalization(axis=-1)(temp) temp = Activation('relu')(temp) temp = Conv2D(3, (9, 9), strides=1, padding='SAME', name='CONV_5', kernel_initializer=init)(temp) temp = Activation('tanh')(temp) temp = Lambda(lambda x: 0.5 * x + 0.5, output_shape=lambda x: x)(temp) return Model(inputs=image, outputs=temp, name=name)
def miccai2018_net(vol_size, enc_nf, dec_nf, int_steps=7, use_miccai_int=False, indexing='ij', bidir=False, vel_resize=1 / 2): """ architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper. You may need to modify this code (e.g., number of layers) to suit your project needs. The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons. :param vol_size: volume size. e.g. (256, 256, 256) :param enc_nf: list of encoder filters. right now it needs to be 1x4. e.g. [16,32,32,32] :param dec_nf: list of decoder filters. right now it must be 1x6, see unet function. :param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration note that the 'velocity' field outputted in that case was since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt **This param will be phased out (set to False behavior)** :param int_steps: the number of integration steps :param indexing: xy or ij indexing. we recommend ij indexing if training from scratch. miccai 2018 runs were done with xy indexing. **This param will be phased out (set to 'ij' behavior)** :return: the keras model """ ndims = len(vol_size) assert ndims in [1, 2, 3], "ndims should be one of 1, 2, or 3. found: %d" % ndims # get unet unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False) [src, tgt] = unet_model.inputs x_out = unet_model.outputs[-1] # velocity mean and logsigma layers Conv = getattr(KL, 'Conv%dD' % ndims) flow_mean = Conv(ndims, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow')(x_out) # we're going to initialize the velocity variance very low, to start stable. flow_log_sigma = Conv( ndims, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10), bias_initializer=keras.initializers.Constant(value=-10), name='log_sigma')(x_out) flow_params = concatenate([flow_mean, flow_log_sigma]) # velocity sample flow = Sample(name="z_sample")([flow_mean, flow_log_sigma]) # integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field) if use_miccai_int: # for the miccai2018 submission, the squaring layer # scaling was essentially built in by the network # was manually composed of a Transform and and Add Layer. v = flow for _ in range(int_steps): v1 = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([v, v]) v = keras.layers.add([v, v1]) flow = v else: # new implementation in neuron is cleaner. z_sample = flow flow = nrn_layers.VecInt(method='ss', name='flow-int', int_steps=int_steps)(z_sample) if bidir: rev_z_sample = Negate()(z_sample) neg_flow = nrn_layers.VecInt(method='ss', name='neg_flow-int', int_steps=int_steps)(rev_z_sample) # get up to final resolution flow = trf_resize(flow, vel_resize, name='diffflow') if bidir: neg_flow = trf_resize(neg_flow, vel_resize, name='neg_diffflow') # transform y = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([src, flow]) if bidir: y_tgt = nrn_layers.SpatialTransformer( interp_method='linear', indexing=indexing)([tgt, neg_flow]) # prepare outputs and losses outputs = [y, flow_params] if bidir: outputs = [y, y_tgt, flow_params] # build the model return Model(inputs=[src, tgt], outputs=outputs)
if isinstance(__version__, (list, tuple)): version_str = ".".join([str(n) for n in __version__[1:]]) else: version_str = __version__ mswindows = sys.platform == "win32" class EncoderType(enum.Enum): ORIGINAL = "original" SHAOANLU = "shaoanlu" ENCODER = EncoderType.ORIGINAL conv_init = RandomNormal(0, 0.02) def inst_norm(): return InstanceNormalization() ENCODER = EncoderType.ORIGINAL hdf = { 'encoderH5': 'encoder_{version_str}{ENCODER.value}.h5'.format(**vars()), 'decoder_AH5': 'decoder_A_{version_str}{ENCODER.value}.h5'.format(**vars()), 'decoder_BH5': 'decoder_B_{version_str}{ENCODER.value}.h5'.format(**vars()) }
def _create_bias_layer(self, inputLayer): userCount = self.params["userCount"] itemCount = self.params["itemCount"] latentFeatures = self.params["latentFeatures"] regularizationScale = self.params["regularizationScale"] userInputLayer = inputLayer["userLayer"] itemInputLayer = inputLayer["itemLayer"] # Define user Bias userBiasLayer = Embedding(input_dim=userCount, output_dim=1, input_length=1, embeddings_initializer=RandomNormal( ), embeddings_regularizer=l2(regularizationScale))(userInputLayer) userBiasLayer = Flatten()(userBiasLayer) # Define item Bias itemBiasLayer = Embedding(input_dim=itemCount, output_dim=1, input_length=1, embeddings_initializer=RandomNormal( ), embeddings_regularizer=l2(regularizationScale))(itemInputLayer) itemBiasLayer = Flatten()(itemBiasLayer) return { "userBiasLayer": userBiasLayer, "itemBiasLayer": itemBiasLayer }
def build_discriminator2D(self, model_shape, filters=32, k_size=4, drop=True, rate=0.5, summary=False, model_file=None, name='gan_d_'): """ Create a Discriminator Model using hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: """ Create a Discriminator Model using hyperparameters values defined as follows """ n_rows = model_shape[0] n_cols = model_shape[1] c_dims = model_shape[2] input_shape = (n_rows, n_cols, c_dims) input_layer = Input(shape=input_shape, name=name + 'input') d = self.Conv2D_Block(input_layer, filters, k_size=k_size, name=name + '1', bn=False) # 30x30x32 d = self.Conv2D_Block(d, 2 * filters, k_size=k_size, name=name + '2') # 15x15x64 d = self.Conv2D_Block(d, 4 * filters, k_size=k_size, name=name + '3') # 8x8x128 d = self.Conv2D_Block(d, 8 * filters, strides=1, k_size=k_size, name=name + '4') # 8x8x256 d = Flatten(name=name + 'flatten')(d) if drop: d = Dropout(rate=rate, name=name + 'dropout')(d, training=True) logits = Dense(1, activation='linear', kernel_initializer=RandomNormal(stddev=0.02), name=name + 'dense')(d) out = Activation('sigmoid', name=name + 'sigmoid')(logits) model = Model(inputs=[input_layer], outputs=[out, logits], name='Discriminator') if (summary): model.summary() return model
def _create_embedding_layer(self, userInputLayer, itemInputLayer): userCount = self.params["userCount"] itemCount = self.params["itemCount"] latentFeatures = self.params["latentFeatures"] regularizationScale = self.params["regularizationScale"] # tf.keras.regularizers.l2 #loss = l2 * reduce_sum(square(x)) userEmbeddingLayer = Embedding(input_dim=userCount, output_dim=latentFeatures, input_length=1, embeddings_regularizer=l2(regularizationScale), embeddings_initializer=RandomNormal())(userInputLayer) userEmbeddingLayer = Flatten()(userEmbeddingLayer) itemEmbeddingLayer = Embedding(input_dim=itemCount, output_dim=latentFeatures, input_length=1, embeddings_regularizer=l2(regularizationScale), embeddings_initializer=RandomNormal())(itemInputLayer) itemEmbeddingLayer = Flatten()(itemEmbeddingLayer) return { "userLayer": userEmbeddingLayer, "itemLayer": itemEmbeddingLayer }
def build_encoder2D(self, model_shape, filters=32, k_size=4, z_size=500, drop=True, rate=0.5, bn=True, summary=False, model_file=None, name='gan_e_'): """ Create a Discriminator Model using hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: """ Create a Discriminator Model using hyperparameters values defined as follows """ n_rows = model_shape[0] n_cols = model_shape[1] c_dims = model_shape[2] input_shape = (n_rows, n_cols, c_dims) input_layer = Input(shape=input_shape, name=name + 'input') x = self.Conv2D_Block(input_layer, filters, k_size=k_size, name=name + '1', bn=False) x = self.Conv2D_Block(x, 2 * filters, k_size=k_size, name=name + '2') x = self.Conv2D_Block(x, 4 * filters, k_size=k_size, name=name + '3') x = self.Conv2D_Block(x, 8 * filters, strides=1, k_size=k_size, name=name + '4') x = Flatten(name=name + 'flatten')(x) if drop: x = Dropout(rate=rate, name=name + 'dropout')(x) x = Dense(z_size, activation='linear', kernel_initializer=RandomNormal(stddev=0.02), name=name + 'dense')(x) if bn: x = BatchNormalization(center=False, scale=False, name=name + "bn_out")(x) model = Model(inputs=[input_layer], outputs=[x], name='Encoder') if (summary): model.summary() return model
def createModel(args, opt): numNodes = [8, 16, 32, 64, 128, 256, 512] k = (args.kernel_size * 2) + 1 w8s = { "Zeros": Zeros(), "Ones": Ones(), "Constant": Constant(value=0.2), "RandNormal": RandomNormal(mean=0.0, stddev=0.05, seed=0) } model = Sequential() model.add(GaussianNoise(args.gauss_noise, input_shape=(32, 32, 3))) model.add( Conv2D(filters=numNodes[0], kernel_size=(k, k), padding='same', kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convolution 1 model.add(Activation('relu')) # ReLU 1 model.add( Conv2D( filters=numNodes[1], kernel_size=(k, k), padding='same', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convlution 2 model.add(Activation('relu')) # ReLU 2 model.add(Dropout(args.dropout_conv)) # Dropout 1 model.add(MaxPooling2D(pool_size=2)) # Max Pooling 1 # model.add( Conv2D( filters=numNodes[2], kernel_size=(k, k), padding='same', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convolution 3 model.add(Activation('relu')) # ReLU 3 model.add( Conv2D( filters=numNodes[3], kernel_size=(k, k), padding='same', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convolution 4 model.add(Activation('relu')) # ReLU 4 model.add(Dropout(args.dropout_conv)) # Dropout 2 model.add(MaxPooling2D(pool_size=2)) # Max Pooling 2 # model.add( Conv2D( filters=numNodes[4], kernel_size=(k, k), padding='same', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convolution 5 model.add(Activation('relu')) # ReLU 5 model.add( Conv2D( filters=numNodes[5], kernel_size=(k, k), padding='same', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Convolution 6 model.add(Activation('relu')) # ReLU 6 model.add(Dropout(args.dropout_conv)) # Dropout 3 model.add(MaxPooling2D(pool_size=2)) # Max Pooling 3 # model.add(Flatten()) # Flatten model.add( Dense(numNodes[6], kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Dense 1 (FC) model.add(Activation('relu')) # ReLU 8 model.add(Dropout(args.dropout_dense)) # Dropout 4 model.add( Dense( 100, activation='softmax', # kernel_initializer=w8s[args.w8s], bias_initializer=w8s[args.w8s])) # Dense 2 (FC) # #print(model.summary()) #print("Model has {} paramters".format(model.count_params())) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) return model
def build_resnet_generator(self, model_shape, filters=32, k_size=3, last_act='tanh', summary=False, model_file=None, name='gan_g_'): """ Create a Generator Model with hyperparameters values defined as follows """ if (model_file): """ Load pretreined model """ model = self.utils.build_pretrained_model(model_file) if (summary): model.summary() return model else: init = RandomNormal(stddev=0.02) n_rows = model_shape[0] n_cols = model_shape[1] in_c_dims = model_shape[2] out_c_dims = model_shape[3] n_rows_e1, n_rows_e2, n_rows_e4, n_rows_e8 = n_rows // 1, n_rows // 2, n_rows // 4, n_rows // 8 rows_matching = np.equal( [2 * n_rows_e2, 2 * n_rows_e4, 2 * n_rows_e8], [n_rows_e1, n_rows_e2, n_rows_e4]) index_rows = np.where(np.logical_not(rows_matching))[0] n_cols_e1, n_cols_e2, n_cols_e4, n_cols_e8 = n_cols // 1, n_cols // 2, n_cols // 4, n_cols // 8 cols_matching = np.equal( [2 * n_cols_e2, 2 * n_cols_e4, 2 * n_cols_e8], [n_cols_e1, n_cols_e2, n_cols_e4]) index_cols = np.where(np.logical_not(cols_matching))[0] input_shape = (n_rows, n_cols, in_c_dims) input_layer = Input(shape=input_shape, name=name + '_input') e1 = self.Conv2D_Block(input_layer, n_kernels=filters, k_size=7, strides=1, bn=False, name=name + 'e1') # rows, cols e2 = self.Conv2D_Block(e1, 2 * filters, k_size=k_size, bn_training=True, name=name + 'e2') # rows/2, cols/2 e3 = self.Conv2D_Block(e2, 4 * filters, k_size=k_size, bn_training=True, name=name + 'e3') # rows/4, cols/4 e4 = self.Conv2D_Block(e3, 8 * filters, k_size=k_size, bn=False, name=name + 'e4') # rows/8, cols/8 rb1 = self.residual_block(e4, n_kernels=8 * filters, k_size=k_size, bn_training=True, name=name + '1_') rb2 = self.residual_block(rb1, n_kernels=8 * filters, k_size=k_size, bn_training=True, name=name + '2_') rb3 = self.residual_block(rb2, n_kernels=8 * filters, k_size=k_size, bn_training=True, name=name + '3_') rb3 = Dropout(rate=0.5, name=name + 'drop_1')(rb3, training=True) rb4 = self.residual_block(rb3, n_kernels=8 * filters, k_size=k_size, bn_training=True, name=name + '4_') rb4 = Dropout(rate=0.5, name=name + 'drop_2')(rb4, training=True) rb5 = self.residual_block(rb4, n_kernels=8 * filters, k_size=k_size, bn_training=True, name=name + '5_') rb5 = Dropout(rate=0.5, name=name + 'drop_3')(rb5, training=True) d1 = self.Conv2DTranspose_Block(rb5, 4 * filters, k_size=k_size, activation='linear', name=name + 'd1') # rows/4, cols/4 if index_rows == 2 or index_cols == 2: d1 = BilinearUpsampling(output_size=(n_rows // 4, n_cols // 4), name=name + '_bilinear')(d1) d1 = Concatenate(name=name + 'conc_1')([d1, e3]) d1 = Activation('relu', name=name + '_act_1')(d1) d2 = self.Conv2DTranspose_Block(d1, 2 * filters, k_size=k_size, activation='linear', name=name + 'd2') # rows/2, cols/2 if index_rows == 1 or index_cols == 1: d2 = BilinearUpsampling(output_size=(n_rows // 2, n_cols // 2), name=name + '_bilinear')(d2) d2 = Concatenate(name=name + 'conc_2')([d2, e2]) d2 = Activation('relu', name=name + '_act_2')(d2) d3 = self.Conv2DTranspose_Block(d2, 1 * filters, k_size=k_size, activation='linear', name=name + 'd3') # rows, cols if index_rows == 0 or index_cols == 0: d3 = BilinearUpsampling(output_size=(n_rows, n_cols), name=name + '_bilinear')(d2) d3 = Concatenate(name=name + 'conc_3')([d3, e1]) d3 = Activation('relu', name=name + 'act_3')(d3) output = Conv2DTranspose(out_c_dims, 7, strides=1, padding='same', kernel_initializer=init, name=name + 'd_out')(d3) # rows, cols # output = InstanceNormalization(axis=-1, name=name+'ins_norm')(output) output = Activation(last_act, name=name + last_act)(output) model = Model(inputs=[input_layer], outputs=[output], name='Generator' + name[-3:]) if (summary): model.summary() return model
ave_acc = 0 ave_auc = 0 for train, test in GroupKFold(n_splits=n_splits).split(X, y, groups=groups): # referring to inputs and outputs: X(train), y(train), X(test), y(test) mcnn = Sequential() mcnn.add( Embedding(vocab_size, EMBEDDING_DIM, input_length=seq_len, embeddings_initializer=Constant(embedding_matrix))) mcnn.add( Conv1D(filters=20, kernel_size=(8), kernel_initializer=RandomNormal(mean=0.0, stddev=0.05))) mcnn.add(MaxPooling1D(pool_size=2)) mcnn.add( Conv1D(filters=20, kernel_size=(5), kernel_initializer=RandomNormal(mean=0.0, stddev=0.05))) mcnn.add(MaxPooling1D(pool_size=2)) mcnn.add( Conv1D(filters=20, kernel_size=(2), kernel_initializer=RandomNormal(mean=0.0, stddev=0.05))) mcnn.add(MaxPooling1D(pool_size=2)) mcnn.add(Flatten()) mcnn.add(Dense(300, activation='relu')) mcnn.add(Dropout(0.5)) mcnn.add(Dense(1, activation='sigmoid'))
netGBH5 = 'netGB_GAN.h5' netDAH5 = 'netDA_GAN.h5' netDBH5 = 'netDB_GAN.h5' netDA2H5 = 'netDA_GAN.h5' netDB2H5 = 'netDB_GAN.h5' netD_codeH5 = 'netD_code_GAN.h5' def __conv_init(a): print("conv_init", a) k = RandomNormal(0, 0.02)(a) # for convolution kernel k.conv_weight = True return k conv_init = RandomNormal(0, 0.02) gamma_init = RandomNormal(1., 0.02) # for batch normalization class GANModel(): img_size = 64 channels = 3 img_shape = (img_size, img_size, channels) encoded_dim = 1024 nc_in = 3 # number of input channels of generators nc_D_inp = 6 # number of input channels of discriminators def __init__(self, model_dir): self.model_dir = model_dir optimizer = Adam(1e-4, 0.5)
def generator_network(name): def resnet_block(n_filters, input_layer): # weight initialization init = RandomNormal(stddev=0.02) # first layer convolutional layer g = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer=init)(input_layer) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) # second convolutional layer g = Conv2D(n_filters, (3, 3), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) # concatenate merge channel-wise with input layer g = Add()([g, input_layer]) return g init = RandomNormal(stddev=0.02) image = Input((64, 64, 3)) y = Lambda(lambda x: 2.0 * x - 1.0, output_shape=lambda x: x)(image) g = Conv2D(64, (7, 7), padding='same', kernel_initializer=init)(y) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) # d128 g = Conv2D(128, (3, 3), strides=(2, 2), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) # d256 g = Conv2D(256, (3, 3), strides=(2, 2), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) for _ in range(3): g = resnet_block(256, g) # u128 g = Conv2DTranspose(128, (3, 3), strides=(2, 2), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) # u64 g = Conv2DTranspose(64, (3, 3), strides=(2, 2), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) g = Activation('relu')(g) # c7s1-3 g = Conv2D(3, (7, 7), padding='same', kernel_initializer=init)(g) g = InstanceNormalization(axis=-1)(g) out_image = Activation('tanh')(g) out_image = Lambda(lambda x: 0.5 * x + 0.5, output_shape=lambda x: x)(out_image) # define model model = Model(inputs=image, outputs=out_image, name=name) return model
def mod_testing8(inp: Union[Tensor, Layer], kernel_initializer: Initializer = RandomNormal(stddev=0.02)): m = GaussianNoise(0.2)(inp) m = conv_layer(m, 16, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 16, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 16, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 32, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 32, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 64, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 64, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 128, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 256, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 256, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 512, kernel_size=3, strides=1, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = conv_layer(m, 512, kernel_size=3, strides=2, dropout=None, batch_norm=None, act="leaky", use_bias=True, kernel_initializer=kernel_initializer) m = Flatten()(m) m = Dense(128, activation=None)(m) m = LeakyReLU(0.2)(m) return m
def generator_deconv(noise_dim, img_dim, batch_size, model_name="generator_deconv", dset="mnist"): """DCGAN generator based on Deconv2D Args: noise_dim: Dimension of the noise input img_dim: dimension of the image output batch_size: needed to reshape after the deconv2D model_name: model name (default: {"generator_deconv"}) dset: dataset (default: {"mnist"}) Returns: keras model """ assert K.backend() == "tensorflow", "Deconv not implemented with theano" s = img_dim[1] f = 512 if dset == "mnist": start_dim = int(s / 4) nb_upconv = 2 elif dset == "celebA": start_dim = int(s / 16) nb_upconv = 4 else: o = s nb_upconv = 0 while o > 7: o = o/2 nb_upconv += 1 start_dim = int(o) reshape_shape = (start_dim, start_dim, f) bn_axis = -1 output_channels = img_dim[-1] gen_input = Input(shape=noise_dim, name="generator_input") # Noise input and reshaping x = Dense(f * start_dim * start_dim, input_dim=noise_dim, use_bias=False)(gen_input) x = Reshape(reshape_shape)(x) x = BatchNormalization(axis=bn_axis)(x) x = Activation("relu")(x) # Transposed conv blocks: Deconv2D->BN->ReLU for i in range(nb_upconv - 1): nb_filters = int(f / (2 ** (i + 1))) s = start_dim * (2 ** (i + 1)) o_shape = (batch_size, s, s, nb_filters) x = Deconv2D(nb_filters, (3, 3), output_shape=o_shape, strides=(2, 2), padding="same", use_bias=False, kernel_initializer=RandomNormal(stddev=0.02))(x) x = BatchNormalization(axis=-1)(x) x = Activation("relu")(x) # Last block s = start_dim * (2 ** (nb_upconv)) o_shape = (batch_size, s, s, output_channels) x = Deconv2D(output_channels, (3, 3), output_shape=o_shape, strides=(2, 2), padding="same", use_bias=False, kernel_initializer=RandomNormal(stddev=0.02))(x) x = Activation("tanh")(x) generator_model = Model(inputs=[gen_input], outputs=[x], name=model_name) visualize_model(generator_model) return generator_model
def model_initialisation(N_input, N_classes, initialiser, seed_value, path_init): """ Model initialization according to a scheme given as input Input: ~ N_input, N_classes Integers ~ initialiser string, among 'normal', 'orth', 'glorot' ~ seed_value seed for reproducibility, it specifies the directory in which to store the results ~ path_init path to store the initialised model Returns: ~ model keras.Models.Sequential instance. A linear stack of layers. Parameters are initialised according to the scheme specified Note that the task to be performed by the network are straight-forward. An equally overall simple model and algorithmic setup suffices to capture the problem complexity """ print("\nModel initialisation. Scheme:") model = Sequential() if (initialiser == 'orth'): print("Orthogonal weights initialisation") weights_initializer = Orthogonal(gain = 1.0, seed = seed_value) elif (initialiser == 'normal'): print("Normal weights initialisation") weights_initializer = RandomNormal(mean = 0.0, stddev = 0.1, seed = seed_value) elif (initialiser == 'glorot'): print("Glorot weights initialisation") weights_initializer = glorot_normal(seed = seed_value) elif (initialiser == 'zeros'): weights_initializer = Zeros() else: print('NO initialiser match') #end model.add(Dense(input_dim = N_input, units = 20, kernel_initializer = weights_initializer, bias_initializer = RandomNormal(mean = 0.0, stddev = 0.1, seed = seed_value), activation = 'relu')) model.add(Dense(input_dim = 20, units = 10, kernel_initializer = weights_initializer, bias_initializer = RandomNormal(mean = 0.0, stddev = 0.1, seed = seed_value), activation = 'relu')) model.add(Dense(units = N_classes, kernel_initializer = weights_initializer, bias_initializer = RandomNormal(mean = 0.0, stddev = 0.1, seed = seed_value), activation = 'softmax')) """ The optimization algorithm details are defined here once for all, the model is returned with these details embedded yet. Hereafter, in the actual training stage it is ready to use with the model.fit(args) method """ sgd = keras.optimizers.SGD(lr = 0.01, decay = 1e-6, momentum = 0.6, nesterov = True) model.compile(loss = 'categorical_crossentropy', optimizer = sgd, metrics = ['accuracy']) streams.check_create_directory(path_init + r'\init') model.save(path_init + r'\init' + r'\model_init.h5') return model
def generator_upsampling(noise_dim, img_dim, model_name="generator_upsampling", dset="mnist"): """DCGAN generator based on Upsampling and Conv2D Args: noise_dim: Dimension of the noise input img_dim: dimension of the image output model_name: model name (default: {"generator_upsampling"}) dset: dataset (default: {"mnist"}) Returns: keras model """ s = img_dim[1] f = 512 if dset == "mnist": start_dim = int(s / 4) nb_upconv = 2 elif dset == "celebA": start_dim = int(s / 16) nb_upconv = 4 else: o = s nb_upconv = 0 while o > 7: o = o/2 nb_upconv += 1 start_dim = int(o) if K.image_dim_ordering() == "th": bn_axis = 1 reshape_shape = (f, start_dim, start_dim) output_channels = img_dim[0] else: reshape_shape = (start_dim, start_dim, f) bn_axis = -1 output_channels = img_dim[-1] gen_input = Input(shape=noise_dim, name="generator_input") # Noise input and reshaping x = Dense(f * start_dim * start_dim, input_dim=noise_dim)(gen_input) x = Reshape(reshape_shape)(x) x = BatchNormalization(axis=bn_axis)(x) x = Activation("relu")(x) # Upscaling blocks: Upsampling2D->Conv2D->ReLU->BN->Conv2D->ReLU for i in range(nb_upconv): x = UpSampling2D(size=(2, 2))(x) nb_filters = int(f / (2 ** (i + 1))) x = Conv2D(nb_filters, (3, 3), padding="same", kernel_initializer=RandomNormal(stddev=0.02))(x) x = BatchNormalization(axis=1)(x) x = Activation("relu")(x) x = Conv2D(nb_filters, (3, 3), padding="same", kernel_initializer=RandomNormal(stddev=0.02))(x) x = Activation("relu")(x) # Last Conv to get the output image x = Conv2D(output_channels, (3, 3), name="gen_conv2d_final", padding="same", activation='tanh', kernel_initializer=RandomNormal(stddev=0.02))(x) generator_model = Model(inputs=[gen_input], outputs=[x], name=model_name) visualize_model(generator_model) return generator_model
height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False # randomly flip images ) datagen.fit(x_train) # ### Model # In[4]: base_model = VGG19(weights='imagenet') # In[5]: conv_init = RandomNormal(stddev=0.03) fc_init = RandomNormal(stddev=0.01) # In[6]: img_input = Input(shape=(32, 32, 3)) # get layers from VGG19 x = base_model.layers[1](img_input) for layer in base_model.layers[2:21]: x = layer(x) # add layers by ourselves x = Flatten()(x) x = Dense(4096, activation='relu', name='fc6', kernel_initializer=fc_init)(x) x = Dropout(0.5)(x) x = base_model.layers[24](x) x = Dropout(0.5)(x)
n_dim_style = 8 n_resblocks = 4 # number of residual blocks in decoder and content encoder n_adain = 2*n_resblocks n_dim_adain = 256 nc_base = 64 # Number of channels of the first conv2d of encoder n_downscale_content = 2 # Number of content encoder dowscaling n_dowscale_style = 4 # Number of style encoder dowscaling use_groupnorm = False # else use_layer norm in upscaling blocks w_l2 = 1e-4 # L2 weight regularization # Optimization configs use_lsgan = True use_mixup = True mixup_alpha = 0.2 batchSize = 1 # was 4 conv_init_dis = RandomNormal(0, 0.02) # initializer of dicriminators' conv layers conv_init = 'he_normal' # initializer of generators' conv layers lrD = 0.00001 # Discriminator learning rate lrG = 0.00001 # Generator learning rate opt_decay = 0 # Learning rate decay over each update. TOTAL_ITERS = 300000 # Max training iterations # Loss weights for generators w_D = 1 # Adversarial loss (Gan) lambda_s=1; # Latent codes- style lambda_c=1; # Latent codes- content luma_w_ir=7; chrome_w_ir=3; luma_w_vis=7; chrome_w_vis=4;
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.input_shape = (128, 128, 3) self.encoder_dim = 512 if self.low_mem else 1024 self.kernel_initializer = RandomNormal(0, 0.02)