from keras import Input from keras.engine import Model from keras.initializers import RandomNormal from keras.layers import Conv2D, BatchNormalization, Dense, Flatten, Dropout, PReLU from util.clipweights import WeightClip weight_init = RandomNormal(mean=0., stddev=0.02) def Conv2DBatchNormPReLU(input_layer, filters, kernel_size, strides=(1, 1), padding='valid'): res = input_layer res = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, activation=None, padding=padding, kernel_initializer=weight_init, kernel_constraint=WeightClip(-0.01, 0.01), bias_constraint=WeightClip(-0.01, 0.01))(res) res = BatchNormalization()(res) res = PReLU(shared_axes=[1, 2])(res) return res class Critic(Model): def __init__(self, input_shape=(224, 224, 3), output_activation=None, inputs=None, outputs=None, name='Critic'): if inputs and outputs: super(Critic, self).__init__(inputs=inputs, outputs=outputs, name=name) return ''' VGG-like conv filters ''' input_image = Input(shape=input_shape) x = input_image
def conv_block(self, x, filters, size, stride=(2,2),has_norm_instance=True, padding='valid', has_activation_layer=True, use_leaky_relu=False): x = Conv2D(filters, size, strides = stride, padding=padding, kernel_initializer=RandomNormal(0, 0.02))(x) if has_norm_instance: x = InstanceNormalization(axis=1)(x) if has_activation_layer: if use_leaky_relu: x = LeakyReLU(alpha=0.2)(x) else: x = Activation('relu')(x) return x pass
def build_model(): model = Sequential() # -------- build model -------- # model = Sequential() # Block 1 model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,64)), name='block1_conv1', input_shape=(30,30,60))) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,64)), name='block1_conv2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')) # Block 2 model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,128)), name='block2_conv1')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,128)), name='block2_conv2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')) # Block 3 model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv1')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv3')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,256)), name='block3_conv4')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')) # Block 4 model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv1')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv3')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = get_he_weight(3,512)), name='block4_conv4')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')) # Block 5 model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv1')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv3')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(get_he_weight(3,512)), name='block5_conv4')) model.add(BatchNormalization()) model.add(Activation('relu')) # model modification for cifar-10 model.add(Flatten(name='flatten')) model.add(Dense(4096, use_bias = True, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='fc_cifa10')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout(dropout)) model.add(Dense(4096, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='fc2')) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout(dropout)) model.add(Dense(10, kernel_regularizer=keras.regularizers.l2(weight_init), kernel_initializer=RandomNormal(stddev = 0.01), name='predictions_cifa10')) model.add(Flatten()) model.add(Dense(1)) # optimizers should be tested # sgd + momentum # others adam = optimizers.Adam(lr=0.0035, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) # sgd = optimizers.SGD(lr=0.005, momentum=0.9, decay=1e-6, nesterov=True) # rms = optimizers.RMSprop(lr=0.0035, rho=0.9, epsilon=1e-08, decay=1e-6) model.compile(optimizer=adam, loss='mse') return model
def rigid_net(vol_size, enc_nf, dec_nf): """ architecture for rigid registration. rigid registration: CNN output ND x ND+1 affine matrix, affine matrix is passed to affine_to_shift function :param vol_size: volume size :param enc_nf: list of encoder :param dec_nf: list of decoder :return: model """ unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False) [src, tgt] = unet_model.inputs x_out = unet_model.outputs[-1] # affine transform matrix # build full connected layer into the model, output the ND x ND+1 affine matrix flow = Conv3D(3, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow')(x_out) #flow = LeakyReLU(0.2)(flow) flow1 = Lambda(reduce_dim1)(flow) flow1 = Lambda(my_reshape)(flow1) print("flow1's shape is: " + str(flow1.shape)) flow2 = Lambda(reduce_dim2)(flow) flow2 = Lambda(my_reshape)(flow2) flow3 = Lambda(reduce_dim3)(flow) flow3 = Lambda(my_reshape)(flow3) # add convolutinal layer into the model, which outputs affine matrix. affine_matrix1 = Conv3D(filters=4, kernel_size=(80, 96, 112), padding='valid', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow1')(flow1) affine_matrix1 = Reshape((1, 4))(affine_matrix1) #affine_matrix1 = LeakyReLU(0.2)(affine_matrix1) print("affine_matrix1's shape is: " + str(affine_matrix1.shape)) affine_matrix2 = Conv3D(filters=4, kernel_size=(80, 96, 112), padding='valid', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow2')(flow2) affine_matrix2 = Reshape((1, 4))(affine_matrix2) #affine_matrix2 = LeakyReLU(0.2)(affine_matrix2) affine_matrix3 = Conv3D(filters=4, kernel_size=(80, 96, 112), padding='valid', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow3')(flow3) affine_matrix3 = Reshape((1, 4))(affine_matrix3) #affine_matrix3 = LeakyReLU(0.2)(affine_matrix3) affine_matrix = concatenate([affine_matrix1, affine_matrix2]) affine_matrix = concatenate([affine_matrix, affine_matrix3]) affine_matrix = Lambda(reduce_dim4)(affine_matrix) print("affine_matrix's shape is :" + str(affine_matrix.shape)) # spatial transform y = nrn_layers.SpatialTransformer(interp_method='linear', indexing='xy')([src, affine_matrix]) model = Model(inputs=[src, tgt], outputs=[y, flow]) print("the output's shape is:" + str(y.shape)) return model
def get_model(configuration='baseline'): AUGMENTATION_CONFIGURATION = configuration ## Model model = Sequential() model.add( Conv2D(32, (3, 3), kernel_initializer='he_normal', bias_initializer=RandomNormal(mean=0.1), input_shape=(16, 16, 1))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(64, (2, 2), kernel_initializer='he_normal', bias_initializer=RandomNormal(mean=0.1))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add( Conv2D(64, (2, 2), kernel_initializer='he_normal', bias_initializer=RandomNormal(mean=0.1))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add( Conv2D(64, (2, 2), kernel_initializer='he_normal', bias_initializer=RandomNormal(mean=0.1))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add( Conv2D(64, (1, 1), kernel_initializer='he_normal', bias_initializer=RandomNormal(mean=0.1))) model.add(LeakyReLU()) model.add(BatchNormalization()) model.add(Conv2D(1, (1, 1), activation='sigmoid')) model.add(Flatten()) # For a binary classification problem sgd = SGD(lr=0.0005, momentum=0.9) model.compile(optimizer=sgd, loss='binary_crossentropy', metrics=['accuracy']) ## Data h5_file_location = os.path.join('C:\\Users\Jeftha\stack\Rommel\ISMI\data', 'prostatex-train.hdf5') h5_file = h5py.File(h5_file_location, 'r') train_data_list, train_labels_list, attr = get_train_data(h5_file, ['ADC']) windowed = [] for lesion in train_data_list: windowed.append(apply_window(lesion, (500, 1100))) train_data_list = np.asarray(windowed) train_data, val_data, train_labels, val_labels = train_test_split( train_data_list, train_labels_list, attr, test_size=0.33) train_data = np.expand_dims(train_data, axis=-1) val_data = np.expand_dims(val_data, axis=-1) train_labels = np.expand_dims(train_labels, axis=-1) val_labels = np.expand_dims(val_labels, axis=-1) ## Stuff for training generator = get_generator(configuration=AUGMENTATION_CONFIGURATION) train_generator = generator.flow( train_data, train_labels) #, save_to_dir="/nfs/home4/schellev/augmented_images") batch_size = 128 steps_per_epoch = len(train_labels_list) // batch_size auc_history = AucHistory(train_data, train_labels, val_data, val_labels, output_graph_name=AUGMENTATION_CONFIGURATION) model.fit_generator(train_generator, steps_per_epoch, epochs=15000, verbose=2, callbacks=[auc_history], max_q_size=50, workers=8) return model
def CNN_LSTM_STATEFUL_model(image_dim, audio_vector_dim): (img_rows, img_cols, img_channels) = image_dim # (224,224,3) input_img = Input(shape=(img_rows, img_cols, img_channels)) # Like Hanoi's work with DeepMind Reinforcement Learning, build a model that does not use pooling layers # to retain sensitivty to locations of objects # Tried (64,128,256,512) x = Conv2D(filters=32, kernel_size=(16, 16), input_shape=image_dim, name='Input_Layer', activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), strides=(8, 8))(input_img) x = Conv2D(filters=64, kernel_size=(8, 8), activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), strides=(4, 4))(x) x = Conv2D(filters=128, kernel_size=(4, 4), activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), strides=(2, 2))(x) x = Conv2D(filters=256, kernel_size=(2, 2), input_shape=image_dim, activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), strides=(1, 1))(x) x = Flatten()(x) x = Dense(1024, activation='relu', name='1st_FC')(x) x = Dropout(0.2)(x) x = Dense(512, activation='relu', name='2nd_FC')(x) # x = TimeDistributedDense(1)(x) # Note that LSTM expects input shape: (nb_samples, timesteps, feature_dim) x = Reshape((1, 512))(x) x = LSTM(256, input_shape=(1, 512), dropout=0.2, return_sequences=True)(x) x = Dropout(0.2)(x) x = LSTM(128, dropout=0.2, name='LSTM_reg_output')(x) network_output = Dense(audio_vector_dim)(x) model = Model(inputs=input_img, outputs=network_output) # Use the Adam optimizer for gradient descent adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(loss='mean_squared_error', optimizer='adam') print(model.summary()) return model
def writers_adversarial_network(class_num, writers_num): """ Our WSAN :param class_num: The total number of chinese character :param writers_num: :return: """ # init softmax random_normal = RandomNormal(stddev=0.001, seed=1995) reg = 1e-9 top5_acc = functools.partial(top_k_categorical_accuracy, k=5) top5_acc.__name__ = 'top5_acc' inputs = Input(shape=(64, 64, 1)) x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(inputs) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(80, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # x = Dropout(0.1)(x) x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(76, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # x = Dropout(0.1)(x) x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(108, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(152, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) gap1 = GlobalAveragePooling2D()(x) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # x = Dropout(0.2)(x) x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(192, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) # x = Dropout(0.2)(x) x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer=glorot_uniform(), use_bias=False, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) x = BatchNormalization()(x) x = Activation('relu')(x) gap2 = GlobalAveragePooling2D()(x) comb_gap = layers.Concatenate(axis=-1)([gap1, gap2]) source_classifier = Dropout(0.5)(comb_gap) source_classifier = Dense(class_num, activation='softmax', name='CR', kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))( source_classifier) grl = GradientReversal(hp_lambda=0.01)(comb_gap) # grl domain_classifier = Dropout(0.5)(grl) domain_classifier = Dense(writers_num, activation='softmax', name='SR', kernel_initializer=random_normal, kernel_regularizer=l2(reg), bias_regularizer=l2(reg))( domain_classifier) opt = optimizers.Adadelta(lr=0.3, rho=0.95, epsilon=None, decay=1e-5) comb_model = Model(inputs=inputs, outputs=[source_classifier, domain_classifier]) # comb_model.summary() # 设置多GPU comb_model = multi_gpu_model(comb_model, gpus=2) comb_model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy', top5_acc]) source_model = Model(inputs=inputs, outputs=[source_classifier]) source_model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy', top5_acc]) domain_model = Model(inputs=inputs, outputs=[domain_classifier]) domain_model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy']) return comb_model, source_model, domain_model
def build_generator(img_shape, gf): """U-Net Generator""" def conv3d(layer_input, filters, f_size=(4, 4, 4), bn=True): d = create_convolution_block(input_layer=layer_input, n_filters=filters, batch_normalization=bn, strides=(2, 2, 2), kernel_size=f_size) return d def deconv3d(layer_input, skip_input, filters, f_size=(4, 4, 4), drop=True): """Layers used during upsampling""" u = create_convolution_block_up(input_layer=layer_input, skip_conn=skip_input, n_filters=filters, batch_normalization=True, strides=(2, 2, 2), kernel_size=f_size, dropout=drop) return u # Image input d0 = Input(batch_shape=img_shape) # Downsampling e1 = conv3d(d0, gf, bn=False) # 64 e2 = conv3d(e1, gf * 2) # 128 e3 = conv3d(e2, gf * 4) # 256 e4 = conv3d(e3, gf * 8) # 512 e5 = conv3d(e4, gf * 8) # 512 # bottleneck e6 = bottleneck(e5, gf * 8, batch_normalization=False, kernel_size=(4, 4, 4), activation='relu', padding='same', strides=(2, 2, 2), instance_normalization=False) # 512 # Upsampling u1 = deconv3d(e6, e5, gf * 8, drop=True) u2 = deconv3d(u1, e4, gf * 8, drop=True) u3 = deconv3d(u2, e3, gf * 4, drop=True) u4 = deconv3d(u3, e2, gf * 2, drop=False) u5 = deconv3d(u4, e1, gf, drop=False) # # init = RandomNormal(mean=0.0, stddev=0.02) # new u6 = Conv3DTranspose(filters=gf, kernel_size=(4, 4, 4), padding='same', kernel_initializer=init, strides=(2, 2, 2))(u5) # final_convolution = Conv3D(1, (1, 1, 1))(u6) act = Activation('relu')(final_convolution) return Model(d0, act)
from generate_example import generate_random_example from create_model import create_model from loss_func import asymmetric_loss_generator from keras.initializers import RandomNormal from keras.optimizers import adam from keras.regularizers import l2 from keras.models import save_model import numpy as np from typing import List, Union from segment_img import read_img, predict_whole_img, visualize_prediction ROWS = 40 COLS = 40 CHANNELS = 3 K_I = RandomNormal(mean=0.0, stddev=0.2) B_I = RandomNormal(mean=0.0, stddev=0.2) K_R = l2(0.01) B_R = l2(0.01) KERNEL_SIZE = 3 STRIDE = 2 LAYERS = 1 CONV_ACTIVATION = 'tanh' CONNECT_ACTIVATION = 'tanh' BATCH_SIZE = 20 EPOCHS = 70 NEG_RATIO_POS = 100 LEARN_RATE = 0.001
def build_model(): model = Sequential() model.add( Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.01), input_shape=x_train.shape[1:], activation='relu')) model.add( Conv2D(160, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add( Conv2D(96, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')) model.add(Dropout(dropout)) model.add( Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')) model.add(Dropout(dropout)) model.add( Conv2D(192, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add( Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add( Conv2D(10, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(0.0001), kernel_initializer=RandomNormal(stddev=0.05), activation='relu')) model.add(GlobalAveragePooling2D()) model.add(Activation('softmax')) sgd = optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model
def createModel(input_sequence_dim, audio_vector_dim): frame_h, frame_w, channels = input_sequence_dim # (80,80,1) we are going to pass in single greyscale images # ConvLSTM expects an input with shape # (n_frames, width, height, channels) In the example they used n_frames = 15 input_sequence = Input(shape=(None, frame_h, frame_w, channels)) # input to a ConvLSTM2D is of form (samples, time, filters, output_row, output_col)` x = ConvLSTM2D(filters=32, kernel_size=(8, 8), padding='same', strides=(4, 4), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), return_sequences=True)(input_sequence) x = BatchNormalization()(x) x = ConvLSTM2D(filters=64, kernel_size=(4, 4), padding='same', strides=(2, 2), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), return_sequences=True)(x) x = BatchNormalization()(x) x = ConvLSTM2D(filters=64, kernel_size=(3, 3), padding='same', strides=(1, 1), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), return_sequences=True)(x) x = BatchNormalization()(x) ''' x = ConvLSTM2D(filters=64, kernel_size=(2, 2), padding='same', strides=(1, 1), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), return_sequences=True)(x) x = BatchNormalization()(x) ''' # The Conv3D layer causes the the hidden "volumes" to become a flat image # The 3D convolution changes a (None, None, 40, 40, 40) into a (None,None,40,40,40) # Output of a Conv3D is (samples, filters, new_conv_dim1, new_conv_dim2, new_conv_dim3) x = Conv3D(filters=1, kernel_size=(3, 3, 3), activation='sigmoid', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), data_format='channels_last')(x) ''' # Run a CNN over the "flat" image x = Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 1), kernel_initializer=RandomNormal(mean=0.0, stddev=0.01, seed=None), padding='same', activation='relu')(x) ''' x = Flatten()(x) # x = Dense(1024, activation='relu', name='1st_FC')(x) # x = Dropout(0.5)(x) x = Dense(512, activation='relu')(x) network_output = Dense(audio_vector_dim, name='regression_out')(x) model = Model(inputs=input_sequence, outputs=network_output) # Use the Adam optimizer for gradient descent adam = Adam(lr=0.5e-6) model.compile(loss='mse', optimizer=adam)
def define_discriminator(image_shape): # weight initialization init = RandomNormal(stddev=0.02) # source image input in_src_image = Input(shape=image_shape) # target image input in_target_image = Input(shape=image_shape) # concatenate images channel-wise merged = Concatenate()([in_src_image, in_target_image]) # C64 d = Conv2D(filters=64, kernel_size=(4, 4), strides=(2, 2), padding='same', kernel_initializer=init)(merged) d = LeakyReLU(alpha=0.2)(d) # C128 d = Conv2D(filters=128, kernel_size=(4, 4), strides=(2, 2), padding='same', kernel_initializer=init)(d) d = BatchNormalization()(d) d = LeakyReLU(alpha=0.2)(d) # C256 d = Conv2D(filters=256, kernel_size=(4, 4), strides=(2, 2), padding='same', kernel_initializer=init)(d) d = BatchNormalization()(d) d = LeakyReLU(alpha=0.2)(d) # C512 d = Conv2D(filters=512, kernel_size=(4, 4), strides=(2, 2), padding='same', kernel_initializer=init)(d) d = BatchNormalization()(d) d = LeakyReLU(alpha=0.2)(d) # second last output layer d = Conv2D(filters=512, kernel_size=(4, 4), padding='same', kernel_initializer=init)(d) d = BatchNormalization()(d) d = LeakyReLU(alpha=0.2)(d) # patch output d = Conv2D(filters=1, kernel_size=(4, 4), padding='same', kernel_initializer=init)(d) patch_out = Activation('sigmoid')(d) # define model model = Model([in_src_image, in_target_image], patch_out) # compile model opt = Adam(lr=0.0002, beta_1=0.5) model.compile(loss='binary_crossentropy', optimizer=opt, loss_weights=[0.5]) return model
def train(tr_features, tr_labels, ts_features, ts_labels): # Hyperparamters epochs = 2000 batch_size = 16 n_dim = tr_features.shape[1] print(n_dim) n_classes = len(cfg.classes) n_hidden_units_one = 30 * n_dim n_hidden_units_two = 20 * n_dim n_hidden_units_three = 20 * n_dim n_hidden_units_four = n_dim n_hidden_units_five = n_dim // 2 n_hidden_units_six = n_dim // 4 sd = 1 / np.sqrt(n_dim) lr = 0.0001 patience = 20 # Build and compile model model = Sequential() # model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_one, input_dim=n_dim, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_two, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_three, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_four, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_five, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_hidden_units_six, activation='sigmoid', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) model.add(Dropout(0.2)) model.add( Dense(n_classes, activation='softmax', kernel_initializer=RandomNormal(stddev=sd), bias_initializer=RandomNormal(stddev=sd))) adam = Adam(lr=lr) # sgd = model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) # Callbacks earlyStopping = EarlyStopping(monitor='val_acc', patience=patience, verbose=1) weights_file = os.path.join(cfg.save_dir, 'weights.{epoch:02d}-{val_acc:.2f}.hdf5') modelCheckpt = ModelCheckpoint(weights_file, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True) tsboard = TensorBoard(log_dir=os.path.join(cfg.save_dir, 'logs'), batch_size=batch_size) # model.load_weights("/data/juliej/results/weights.34-0.81.hdf5") # Train model.fit( tr_features, tr_labels, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(ts_features, ts_labels), shuffle=True, # initial_epoch=35, callbacks=[earlyStopping, modelCheckpt, tsboard])
def __init__(self, n_clusters, input_dim, encoded=None, decoded=None, alpha=1.0, pretrained_weights=None, cluster_centres=None, batch_size=256, **kwargs): super(DeepEmbeddingClustering, self).__init__() self.n_clusters = n_clusters self.input_dim = input_dim self.encoded = encoded self.decoded = decoded self.alpha = alpha self.pretrained_weights = pretrained_weights self.cluster_centres = cluster_centres self.batch_size = batch_size self.learning_rate = 0.1 self.iters_lr_update = 20000 self.lr_change_rate = 0.1 # greedy layer-wise training before end-to-end training: self.encoders_dims = [self.input_dim, 500, 500, 2000, 10] self.input_layer = Input(shape=(self.input_dim, ), name='input') dropout_fraction = 0.2 init_stddev = 0.01 self.layer_wise_autoencoders = [] self.encoders = [] self.decoders = [] for i in range(1, len(self.encoders_dims)): encoder_activation = 'linear' if i == (len(self.encoders_dims) - 1) else 'relu' encoder = Dense(self.encoders_dims[i], activation=encoder_activation, input_shape=(self.encoders_dims[i - 1], ), kernel_initializer=RandomNormal(mean=0.0, stddev=init_stddev, seed=None), bias_initializer='zeros', name='encoder_dense_%d' % i) self.encoders.append(encoder) decoder_index = len(self.encoders_dims) - i decoder_activation = 'linear' if i == 1 else 'relu' decoder = Dense(self.encoders_dims[i - 1], activation=decoder_activation, kernel_initializer=RandomNormal(mean=0.0, stddev=init_stddev, seed=None), bias_initializer='zeros', name='decoder_dense_%d' % decoder_index) self.decoders.append(decoder) autoencoder = Sequential([ Dropout(dropout_fraction, input_shape=(self.encoders_dims[i - 1], ), name='encoder_dropout_%d' % i), encoder, Dropout(dropout_fraction, name='decoder_dropout_%d' % decoder_index), decoder ]) autoencoder.compile(loss='mse', optimizer=SGD(lr=self.learning_rate, decay=0, momentum=0.9)) self.layer_wise_autoencoders.append(autoencoder) # build the end-to-end autoencoder for finetuning # Note that at this point dropout is discarded self.encoder = Sequential(self.encoders) self.encoder.compile(loss='mse', optimizer=SGD(lr=self.learning_rate, decay=0, momentum=0.9)) self.decoders.reverse() self.autoencoder = Sequential(self.encoders + self.decoders) self.autoencoder.compile(loss='mse', optimizer=SGD(lr=self.learning_rate, decay=0, momentum=0.9)) if cluster_centres is not None: assert cluster_centres.shape[0] == self.n_clusters assert cluster_centres.shape[1] == self.encoder.layers[ -1].output_dim if self.pretrained_weights is not None: self.autoencoder.load_weights(self.pretrained_weights)
def _build(self): self.int_dim = self.config['int_features'] self.float_dim = self.config['float_features'] self.feature_sizes = self.config['int_feature_sizes'] self.embed_size = self.config['embed_size'] self.total_feature_size = sum(self.feature_sizes) + self.float_dim self.all_dim = self.int_dim + self.float_dim ################################################################## # Begin Define shared layers ################################################################## self.input_embedding_layer = Lambda(self.build_embedding_input) self.new_input_layer = Lambda(self.build_new_input) # Add Embedding For all features # [None, all_dim, emb_size] from keras.initializers import RandomNormal self.V_embedding_layer = Embedding( self.total_feature_size, self.embed_size, embeddings_initializer=RandomNormal(stddev=0.001)) # [None, all_dim, 1] self.x_embedding_layer = Embedding( self.total_feature_size, 1, embeddings_initializer=RandomNormal(stddev=0.001)) # FM Part self.FM_layer = Lambda(function=self.FM_part) self.dropout_layer1 = Dropout(self.config['input_dropout']) #y_FM = self.FM_part(x_embedding, V_embedding, new_input) self.GLU_layers = [] for i in range(self.config['hidden_layers']): dense = Dense(self.config['hidden_units']) gate = Dense(self.config['hidden_units'], activation='sigmoid') self.GLU_layers.append((dense, gate)) self.dense_DNN_layer = Dense(1, activation="elu") self.concate_layer = Lambda(lambda x: K.concatenate( [K.reshape(x[0], (-1, 1)), K.reshape(x[1], (-1, 1))], axis=1)) self.dense_concat_layer = Dense(1, activation="sigmoid") ################################################################## # End Define shared layers ################################################################## ################################################################## # Build Predict model ################################################################## input = Input(shape=(self.all_dim, )) input_embedding = self.input_embedding_layer(input) new_input = self.new_input_layer(input) # Add Embedding For all features # [None, all_dim, emb_size] V_embedding = self.V_embedding_layer(input_embedding) # [None, all_dim, 1] x_embedding = self.x_embedding_layer(input_embedding) # FM Part y_FM = self.FM_layer([x_embedding, V_embedding, new_input]) # DNN Part flatten = Flatten()(V_embedding) XF = self.dropout_layer1(flatten) for i in range(self.config['hidden_layers']): #GLU dense, gate = self.GLU_layers[i] XF = multiply([dense(XF), gate(XF)]) #以上三步构成了所谓的GLU激活函数 #XF = dense(XF) #Normal dense #input = Dense(self.config['hidden_units'],activation=self.config['hidden_activation'])(input) #input = Dropout(self.config['hidden_dropout'])(input) y_DNN = self.dense_DNN_layer(XF) concated = self.concate_layer([y_DNN, y_FM]) score = self.dense_concat_layer(concated) self.predict_model = Model(inputs=input, outputs=score) self.predict_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', auc]) self.predict_model.summary() ################################################################## # Build Rank model ################################################################## lr = self.config['learning_rate'] lr = 1e-4 print("learning_rate=%f" % (lr)) optimizer = Adam(lr) optimizer = SGD(lr, momentum=0.95, decay=0.99) pos_input = Input(shape=(self.all_dim, ), name="Pos_Inp") neg_input = Input(shape=(self.all_dim, ), name="Neg_Inp") pos_score = self.predict_model(pos_input) neg_score = self.predict_model(neg_input) margin = 0.6 loss = Lambda(lambda x: K.relu(margin - x[0] + x[1]), name="loss")([pos_score, neg_score]) #loss = Lambda(lambda x : K.sigmoid(-x[0] + x[1]),name="loss")([pos_score,neg_score]) """ train_model = Model(inputs=[pos_input,neg_input],outputs=[pos_score,neg_score,loss]) train_model.compile(optimizer=optimizer, loss=['binary_crossentropy','binary_crossentropy',lambda y_true, y_pred: y_pred], loss_weights=[5.0,1.0,10.0], metrics={'loss':'binary_crossentropy'}) """ train_model = Model(inputs=[pos_input, neg_input], outputs=loss) train_model.compile( optimizer=optimizer, loss=lambda y_true, y_pred: y_pred, ) self.model = train_model self.model.summary() plot_model(self.model, "deepfmrank_train_model.png") plot_model(self.predict_model, "deepfmrank_predict_model.png")
max_radius = max_radius, noise_treatment = noise_treatment, weights = weights, scaling_weights = scaling_weights) ##------------------------------ Model ---------------------------------------## # Create model = Sequential() # Add layers model.add( Conv3D(filters=32, kernel_size=9, strides=2, padding='valid', kernel_initializer=RandomNormal(mean=0.0, stddev=stddev_conv3d * 9**(-3 / 2)), bias_initializer='zeros', kernel_regularizer=l2(0.001), bias_regularizer=None, input_shape=(v_size, v_size, v_size, n_channels))) model.add(LeakyReLU(alpha=0.1)) model.add(Dropout(rate=0.2)) model.add( Conv3D(filters=64, kernel_size=5, strides=1, padding='valid', kernel_initializer=RandomNormal(mean=0.0,
def RRDB1(inp: Union[Layer, tf.Tensor], filters: int = 64, kernel_size: int = 3, use_bias: bool = True, use_sn: bool = False, kernel_initializer: Initializer = RandomNormal(stddev=0.02)): ConvLayer = ConvSN2D if use_sn else Conv2D def dense_block(inp): x1 = ConvLayer(filters, kernel_size=kernel_size, strides=1, padding='same', use_bias=use_bias, kernel_initializer=kernel_initializer)(inp) x1 = LeakyReLU(0.2)(x1) x1 = Concatenate()([inp, x1]) x2 = ConvLayer(filters, kernel_size=kernel_size, strides=1, padding='same', use_bias=use_bias, kernel_initializer=kernel_initializer)(x1) x2 = LeakyReLU(0.2)(x2) x2 = Concatenate()([inp, x1, x2]) x3 = ConvLayer(filters, kernel_size=kernel_size, strides=1, padding='same', use_bias=use_bias, kernel_initializer=kernel_initializer)(x2) x3 = LeakyReLU(0.2)(x3) x3 = Concatenate()([inp, x1, x2, x3]) x4 = ConvLayer(filters, kernel_size=kernel_size, strides=1, padding='same', use_bias=use_bias, kernel_initializer=kernel_initializer)(x3) x4 = LeakyReLU(0.2)(x4) x4 = Concatenate()([inp, x1, x2, x3, x4]) x5 = ConvLayer(filters, kernel_size=kernel_size, strides=1, padding='same', use_bias=use_bias, kernel_initializer=kernel_initializer)(x4) x5 = Lambda(lambda x: x * 0.2)(x5) x = Add()([x5, inp]) return x x = dense_block(inp) x = dense_block(x) x = dense_block(x) x = Lambda(lambda x: x * 0.2)(x) out = Add()([x, inp]) return out
from data import _seed import numpy as np np.random.seed(_seed) import sys from keras.layers import Dense, merge, Dropout, RepeatVector, Embedding from keras.layers import recurrent, Input, Activation from keras.models import Model from keras.initializers import RandomNormal, Orthogonal from experiment import evaluate_model, handlesysargs from model import model as modelwrapper print("MTL model sharing everything but output, based on gru") INITIALIZER = RandomNormal(mean=0.0, stddev=0.05, seed=_seed) RINITIALIZER = Orthogonal(gain=1.0, seed=_seed) RNN = recurrent.GRU EMBED_HIDDEN_SIZE = 30 #50 SENT_HIDDEN_SIZE = 100 QUERY_HIDDEN_SIZE = 100 def compile_model(inputs, repeat): (vocab_size, story_maxlen, query_maxlen) = inputs[0] #(vocab_size2, story_maxlen2, query_maxlen2) = inputs[1] #mvocab_size = vocab_size1 #if (vocab_size2 > mvocab_size): # mvocab_size = vocab_size2 ensinput = Input((story_maxlen, )) sentemb = Embedding(vocab_size, EMBED_HIDDEN_SIZE,
def CNN_LSTM_model(image_dim, audio_vector_dim, learning_rate, weight_init): #(224,224,3)timespace-imageセット3枚 ( img_rows, img_cols, img_channels ) = image_dim # (224,224,3)*hoge=extract_cont_TopAngles.pyで最後に繋げた長さがtraining_data数 input_img = Input(shape=(img_rows, img_cols, img_channels)) # Like Hanoi's work with DeepMind Reinforcement Learning, build a model that does not use pooling layers # to retain sensitivty to locations of objects #物体の位置に関する繊細さを得るためにこのモデルはプーリング層を使わないで行った。 #CNN layers that increase in filter number, decrease in filter size # and decrease in filter stride. The authors reasoned # that such a design pattern enables the CNN to be # sensitive to the location of small details # Tried (64,128,256,512) x = Conv2D( filters=32, kernel_size=(16, 16), input_shape=image_dim, name='Input_Layer', activation='relu', padding='same', kernel_initializer=RandomNormal( mean=0.0, stddev=weight_init, seed=None), #正規分布に従って重みを初期化,stddev=weight_init = 0.01 分布の標準偏差 strides=(8, 8))(input_img) x = Conv2D(filters=64, kernel_size=(8, 8), activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=weight_init, seed=None), strides=(4, 4))(x) x = Conv2D(filters=128, kernel_size=(4, 4), activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=weight_init, seed=None), strides=(2, 2))(x) x = Conv2D(filters=256, kernel_size=(2, 2), input_shape=image_dim, activation='relu', padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=weight_init, seed=None), strides=(1, 1))(x) x = Flatten()(x) x = Dense(1024, activation='relu', name='1st_FC', kernel_initializer=RandomNormal(mean=0.0, stddev=weight_init, seed=None))(x) x = Dropout(0.2)(x) x = Dense(512, activation='relu', name='2nd_FC', kernel_initializer=RandomNormal(mean=0.0, stddev=weight_init, seed=None))(x) # x = TimeDistributedDense(1)(x) # Note that LSTM expects input shape: (nb_samples, timesteps, feature_dim) x = Reshape((1, 512))(x) x = LSTM(256, input_shape=(1, 512), dropout=0.2, return_sequences=True)(x) x = Dropout(0.2)(x) x = LSTM(128, dropout=0.2, name='LSTM_reg_output')(x) #network_output = Dense(audio_vector_dim)(x)#最後にオーディオデータの次元数にあわせる network_output = Dense(10)(x) #最後にオーディオデータの次元数にあわせる model = Model(inputs=input_img, outputs=network_output) # Use the Adam optimizer for gradient descent adam = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) #sgd = SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False) print("learning rate:", learning_rate) model.compile(loss='mean_squared_error', optimizer='adam') print(model.summary()) return model
print(x_test.shape[0], 'test samples') # Convert class vectors to binary class matrices. y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() m = 0 sd = 0.05 #use_bias = True, bias_initializer = RandomNormal(mean=0.05, stddev=0.05, seed=None), model.add( Conv2D(192, (5, 5), padding='same', use_bias=True, bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None), input_shape=x_train.shape[1:])) model.add(Activation('relu')) model.add( Conv2D(160, (1, 1), use_bias=True, bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None))) model.add(Activation('relu')) model.add( Conv2D(96, (1, 1), use_bias=True, bias_initializer=RandomNormal(mean=m, stddev=sd, seed=None))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(3, 3), strides=2)) model.add(Dropout(0.5))
import keras.backend as K import tensorflow as tf import numpy as np from keras.layers import Input, Reshape, Flatten, Dense, BatchNormalization, PReLU from keras.layers import Activation, Add, Lambda, AveragePooling2D, LeakyReLU, GlobalAvgPool2D from keras.layers.convolutional import UpSampling2D, Conv2D from keras.models import Model from keras.initializers import RandomNormal, glorot_uniform conv_init = RandomNormal(0, 0.02) # def SubpixelConv2D(input_shape, scale=2): # # # Copyright (c) 2017 Jerry Liu # # Released under the MIT license # # https://github.com/twairball/keras-subpixel-conv/blob/master/LICENSE # # def subpixel_shape(input_shape): # dims = [input_shape[0], # input_shape[1] * scale, # input_shape[2] * scale, # int(input_shape[3] / (scale ** 2))] # output_shape = tuple(dims) # return output_shape # # def subpixel(x): # return tf.depth_to_space(x, scale) # # return Lambda(subpixel, output_shape=subpixel_shape) def Resblock_generator(layer_input, channels):
def __init__(self, BiGRU_rand=True, STATIC=False, ExternalEmbeddingModel=None, n_symbols=None, wordmap=None): self.embedding_dim = 300 self.hidden_dims = 100 self.dropout_prob = (0.5, 0.8) self.loss = 'categorical_crossentropy' self.optimizer = 'rmsprop' self.l1_reg = 0 self.l2_reg = 3 ##according to kim14 self.std = 0.05 ## standard deviation # Training Parameters self.set_training_paramters(batch_size=64, num_epochs=10) self.set_processing_parameters( sequence_length=30, vocab_size=50000) ## changed to fit short text # Defining Model Layers if clstm_rand: ##Embedding Layer Randomly initialized if BiGRU_rand: ##Embedding Layer Randomly initialized embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=self.vocab_size) Classes = dh.read_labels() n_classes = len(Classes) else: ## Use pretrained model #n_symbols, wordmap = dh.get_word_map_num_symbols() self.set_etxrernal_embedding(ExternalEmbeddingModel) vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel) Classes = dh.read_labels() n_classes = len(Classes) ## Define Embedding Layer embedding_weights = dh.GetEmbeddingWeights( embedding_dim=self.embedding_dim, n_symbols=n_symbols, wordmap=wordmap, vecDic=vecDic) embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=n_symbols, trainable=STATIC) embedding_layer.build( (None, )) # if you don't do this, the next step won't work embedding_layer.set_weights([embedding_weights]) Sequence_in = Input(shape=(self.sequence_length, ), dtype='int32') embedding_seq = embedding_layer(Sequence_in) x = Dropout(self.dropout_prob[0])(embedding_seq) x = Bidirectional( GRU(self.hidden_dims, kernel_initializer=RandomNormal(stddev=self.std), kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg), return_sequences=False))(x) preds = Dense(n_classes, activation='softmax')(x) ## return graph model model = Model(Sequence_in, preds) model.compile(loss=self.loss, optimizer=self.optimizer, metrics=['accuracy']) self.model = model
def miccai2018_net(vol_size, enc_nf, dec_nf, use_miccai_int=True, int_steps=7, indexing='xy'): """ architecture for probabilistic diffeomoprhic VoxelMorph presented in the MICCAI 2018 paper. You may need to modify this code (e.g., number of layers) to suit your project needs. The stationary velocity field operates in a space (0.5)^3 of vol_size for computational reasons. :param vol_size: volume size. e.g. (256, 256, 256) :param enc_nf: list of encoder filters. right now it needs to be 1x4. e.g. [16,32,32,32] :param dec_nf: list of decoder filters. right now it must be 1x6, see unet function. :param use_miccai_int: whether to use the manual miccai implementation of scaling and squaring integration note that the 'velocity' field outputted in that case was since then we've updated the code to be part of a flexible layer. see neuron.layers.VecInt :param int_steps: the number of integration steps :param indexing: xy or ij indexing. we recommend ij indexing if training from scratch. miccai 2018 runs were done with xy indexing. :return: the keras model """ # get unet unet_model = unet_core(vol_size, enc_nf, dec_nf, full_size=False) [src, tgt] = unet_model.inputs x_out = unet_model.outputs[-1] # velocity mean and logsigma layers flow_mean = Conv3D(3, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5), name='flow')(x_out) flow_log_sigma = Conv3D( 3, kernel_size=3, padding='same', kernel_initializer=RandomNormal(mean=0.0, stddev=1e-10), bias_initializer=keras.initializers.Constant(value=-10), name='log_sigma')(x_out) flow_params = concatenate([flow_mean, flow_log_sigma]) # velocity sample flow = Lambda(sample, name="z_sample")([flow_mean, flow_log_sigma]) # integrate if diffeomorphic (i.e. treating 'flow' above as stationary velocity field) if use_miccai_int: # for the miccai2018 submission, the scaling and squaring layer # was manually composed of a Transform and and Add Layer. flow = Lambda(lambda x: x, name='flow-fix')(flow) # remanant of old code v = flow for _ in range(int_steps): v1 = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([v, v]) v = keras.layers.add([v, v1]) flow = v else: # new implementation in neuron is cleaner. # the 2**int_steps is a correcting factor left over from the miccai implementation. # * (2**int_steps) flow = Lambda(lambda x: x, name='flow-fix')(flow) flow = nrn_layers.VecInt(method='ss', name='flow-int', int_steps=7)(flow) # get up to final resolution flow = Lambda(interp_upsampling, output_shape=vol_size + (3, ), name='pre_diffflow')(flow) flow = Lambda(lambda arg: arg * 2, name='diffflow')(flow) # transform y = nrn_layers.SpatialTransformer(interp_method='linear', indexing=indexing)([src, flow]) # prepare outputs and losses outputs = [y, flow_params] # build the model return Model(inputs=[src, tgt], outputs=outputs)
def __init__(self, att_rand=True, ExternalEmbeddingModel=None, n_symbols=None, wordmap=None, STATIC=True): self.dropout_prob = (0.36, 0.36) self.hidden_dims = 100 self.std = 0.05 self.l1_reg = 3 self.l2_reg = 3 self.loss = 'categorical_crossentropy' self.optimizer = 'rmsprop' self.sequence_length = 30 self.embedding_dim = 300 self.vocab_size = 50000 self.num_epochs = 5 self.batch_size = 64 ## Define the BiGRU model if att_rand: ##Embedding Layer Randomly initialized embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=self.vocab_size) else: ## Use pretrained model #n_symbols, wordmap = dh.get_word_map_num_symbols() self.set_etxrernal_embedding(ExternalEmbeddingModel) vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel) ## Define Embedding Layer embedding_weights = dh.GetEmbeddingWeights( embedding_dim=self.embedding_dim, n_symbols=n_symbols, wordmap=wordmap, vecDic=vecDic) embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=n_symbols, trainable=STATIC) embedding_layer.build( (None, )) # if you don't do this, the next step won't work embedding_layer.set_weights([embedding_weights]) ################################################### SeqIn = Input(shape=(self.sequence_length, ), dtype='int32') embedding_seq = embedding_layer(SeqIn) M1 = Dropout(self.dropout_prob[0])(embedding_seq) #M1 = Activation('tanh')(embedding_seq) activations = Bidirectional( GRU(self.hidden_dims, kernel_initializer=RandomNormal(stddev=self.std), kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg), return_sequences=True))(M1) ## Timedistributed dense for each activation attention = TimeDistributed(Dense(1, activation='tanh'))(activations) attention = Flatten()(attention) attention = Activation('softmax')(attention) attention = RepeatVector(2 * (self.hidden_dims))(attention) attention = Permute([2, 1])(attention) # apply the attention sent_representation = merge([activations, attention], mode='mul') sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))( sent_representation) Classes = dh.read_labels() n_classes = len(Classes) preds = Dense(n_classes, activation='softmax')(sent_representation) model = Model(input=SeqIn, output=preds) model.compile(optimizer=self.optimizer, loss=self.loss, metrics=['accuracy']) self.model = model
other_t: features }) # NUM_CLUSTER * num images # select reliable images reliable_image_idx = np.unique(np.argwhere(similarities > LAMBDA)[:, 1]) print('ckpt %d: # reliable images %d' % (ckpt, len(reliable_image_idx))) sys.stdout.flush() images = np.array(labeled_images + [unlabeled_images[i][0] for i in reliable_image_idx]) labels = to_categorical( known_labels + [kmeans.labels_[i] + KNOWN for i in reliable_image_idx]) # retrain: fine tune init_model = load_model('checkpoint/0.ckpt') x = init_model.get_layer('avg_pool').output x = Flatten(name='flatten')(x) x = Dropout(0.5)(x) x = Dense(NUM_IDS, activation='softmax', name='fc8', kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))(x) net = Model(input=init_model.input, output=x) for layer in net.layers: layer.trainable = True net.compile(optimizer=SGD(lr=0.001, momentum=0.9), loss='categorical_crossentropy') net.fit_generator(datagen.flow(images, labels, batch_size=BATCH_SIZE), steps_per_epoch=len(images) / BATCH_SIZE + 1, epochs=NUM_EPOCH) net.save('checkpoint/%d.ckpt' % ckpt)
def __init__(self, cnn_rand=True, STATIC=False, ExternalEmbeddingModel=None, n_symbols=None, wordmap=None): # Model hyperparameters self.embedding_dim = 300 ## self.filter_sizes = (3, 8) self.num_filters = 10 self.hidden_dims = 100 self.dropout_prob = (0.5, 0.8) self.loss = 'categorical_crossentropy' self.optimizer = 'rmsprop' self.l1_reg = 0 self.l2_reg = 3 ##according to kim14 self.std = 0.05 ## standard deviation # Training Parameters self.set_training_paramters(batch_size=64, num_epochs=10) self.set_processing_parameters( sequence_length=100, vocab_size=vocabsize) ## changed to fit short text # Defining Model Layers if cnn_rand: ##Embedding Layer Randomly initialized embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=self.vocab_size) Classes = dh.read_labels() n_classes = len(Classes) else: ## Use pretrained model #n_symbols, wordmap = dh.get_word_map_num_symbols() self.set_etxrernal_embedding(ExternalEmbeddingModel) vecDic = dh.GetVecDicFromGensim(self.ExternalEmbeddingModel) Classes = dh.read_labels() n_classes = len(Classes) ## Define Embedding Layer embedding_weights = dh.GetEmbeddingWeights(embedding_dim=300, n_symbols=n_symbols, wordmap=wordmap, vecDic=vecDic) embedding_layer = Embedding(output_dim=self.embedding_dim, input_dim=n_symbols, trainable=STATIC) embedding_layer.build( (None, )) # if you don't do this, the next step won't work embedding_layer.set_weights([embedding_weights]) Sequence_in = Input(shape=(self.sequence_length, ), dtype='int32') embedding_seq = embedding_layer(Sequence_in) x = Dropout(self.dropout_prob[0])(embedding_seq) ## define Core Convultional Layers conv_blocks = [] for sz in self.filter_sizes: conv = Convolution1D(filters=self.num_filters, kernel_size=sz, padding="valid", activation="relu", strides=1)(x) conv = MaxPooling1D(pool_size=2)(conv) conv = Flatten()(conv) conv_blocks.append(conv) x = Concatenate()( conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0] x = Dropout(self.dropout_prob[1])(x) x = Dense(self.hidden_dims, activation="relu", kernel_initializer=RandomNormal(stddev=self.std), kernel_regularizer=L1L2(l1=self.l1_reg, l2=self.l2_reg))(x) preds = Dense(n_classes, activation='softmax')(x) ## return graph model model = Model(Sequence_in, preds) model.compile(loss=self.loss, optimizer=self.optimizer, metrics=['accuracy']) self.model = model
def deconv_block(self, x, filters, size ): x = Conv2DTranspose(filters, kernel_size=size, strides=2, padding='same', use_bias=False, kernel_initializer=RandomNormal(0, 0.02))(x) x = InstanceNormalization(axis=1)(x) x = Activation('relu')(x) return x pass
def create_base_network_vgg(input_shape): '''Base network to be shared (eq. to feature extraction). ''' model = Input(shape=input_shape) x = Conv2D(16, (3, 3), padding="same", activation='relu', data_format='channels_first', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(model) x = Conv2D(16, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) #x = Dropout(0.2)(x) x = Conv2D(32, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = Conv2D(32, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) #if stride==None default to pool_size #x = Dropout(0.5)(x) x = Conv2D(64, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = Conv2D(64, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = Conv2D(64, (3, 3), padding="same", activation='relu', kernel_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed), bias_initializer=RandomNormal(mean=0.0, stddev=0.05, seed=random_seed))(x) x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x) #if stride==None default to pool_size x = Flatten()(x) #x = Dense(500, activation='relu')(x) x = Dense( 2048, kernel_initializer=keras.initializers.lecun_normal(seed=random_seed), activation='relu')(x) #x = Dropout(0.1)(x) x = Dense( 2048, kernel_initializer=keras.initializers.lecun_normal(seed=random_seed), activation='relu')(x) #x = Dropout(0.1)(x) x = Dense( 2048, kernel_initializer=keras.initializers.lecun_normal(seed=random_seed), activation='relu')(x) #x = Dropout(0.1)(x) #model = Dense(num_classes)(model) return Model(model, x)
from keras.activations import relu from keras.initializers import RandomNormal from keras.optimizers import RMSprop, SGD, Adam K.set_image_data_format('channels_last') channel_axis = -1 channel_first = False def __conv_init(a): print("conv_init", a) k = RandomNormal(0, 0.02)(a) # for convolution kernel k.conv_weight = True return k conv_init = RandomNormal(0, 0.02) gamma_init = RandomNormal(1., 0.02) # for batch normalization # Basic discriminator def conv2d(f, *a, **k): return Conv2D(f, kernel_initializer=conv_init, *a, **k) def batchnorm(): return BatchNormalization(momentum=0.9, axis=channel_axis, epsilon=1.01e-5, gamma_initializer=gamma_init)
def __conv_init(a): print("conv_init", a) k = RandomNormal(0, 0.02)(a) # for convolution kernel k.conv_weight = True return k