def create_model(self, model_params={ 'n1': 32, 'n2': 64, 'n3': 32, 'frame_len': 80 }): frame_len = self.frame_len n1 = model_params['n1'] n2 = model_params['n2'] n3 = model_params['n3'] input_sque = Input(shape=(frame_len, 1)) c1 = Conv1D(n1, 3, padding='same')(input_sque) c1 = Activation(selu)(c1) c1 = Conv1D(n1, 3, padding='same')(c1) c1 = Activation(selu)(c1) x = MaxPooling1D(2)(c1) c2 = Conv1D(n2, 3, padding='same')(x) c2 = Activation(selu)(c2) c2 = Conv1D(n2, 3, padding='same')(c2) c2 = Activation(selu)(c2) x = MaxPooling1D(2)(c2) c3 = Conv1D(n3, 3, padding='same')(x) c3 = Activation(selu)(c3) x = UpSampling1D(2)(c3) c2_2 = Conv1D(n2, 3, padding='same')(x) c2_2 = Activation(selu)(c2_2) c2_2 = Conv1D(n2, 3, padding='same')(c2_2) c2_2 = Activation(selu)(c2_2) m1 = Add()([c2, c2_2]) m1 = UpSampling1D(2)(m1) c1_2 = Conv1D(n1, 3, padding='same')(m1) c1_2 = Activation(selu)(c1_2) c1_2 = Conv1D(n1, 3, padding='same')(c1_2) c1_2 = Activation(selu)(c1_2) m2 = Add()([c1, c1_2]) decoded = Conv1D(1, 5, padding='same', activation='linear')(m2) model = Model(input_sque, decoded) model.summary() learning_rate = self.learning_rate # adam = optimizers.Adam(lr=learning_rate) # model.compile(optimizer=adam, loss='mse', metrics=[SNRLoss]) adam_wn = AdamWithWeightnorm(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(optimizer=adam_wn, loss='mse', metrics=[snr]) return model
def make_encoder(decoder): latent_dim = decoder.input_shape[-1] h, w, c = decoder.output_shape[-3:] encoder = residual_encoder(h, w, c, latent_dim) encoder.name = 'encoder' for l in decoder.layers: l.trainable = False decoder.trainable = False latent_input = Input(shape=(latent_dim, )) real_input = Input(shape=(h, w, c)) x_h_0 = decoder(latent_input) # latent -> image y_h_0 = encoder(x_h_0) # image -> latent x_h_h_0 = decoder(y_h_0) # latent -> image y_h_1 = encoder(real_input) # image -> latent x_h_1 = decoder(y_h_1) # latent -> image loss = K.mean(K.square(y_h_0 - latent_input)) + K.mean( K.square(x_h_h_0 - x_h_0)) + K.mean(K.square(x_h_1 - real_input)) model = Model([real_input, latent_input], [x_h_1, x_h_h_0]) model.add_loss(loss) model.compile(optimizer=AdamWithWeightnorm(lr=0.0001, beta_1=0.5), loss=None) return model, encoder
def __init__(self,myopt,mylr,NumXgenes,NumYgenes,ArchType): if myopt == 'adam': optimizer = AdamWithWeightnorm(lr=mylr, beta_1=0.9, beta_2=0.999,epsilon=1e-8) elif myopt == 'adadelta': optimizer= Adadelta() else: print('Not a valid optimizer') self.ld_genes_shape = NumXgenes self.target_genes_shape = NumYgenes self.ArchType = ArchType ############################################### # hard coded hyperparameters below self.gen_hidden_units = 9000 self.disc_hidden_units = 3000 self.leak_value = 0.2 ## lower bernoulli p means higher number of zeros self.init_bernoulli_p = 0.1 ## Need to change with each iteration or epoch self.final_bernoulli_p = 0.99 self.u1 = 0.1 self.u2 = 0.15 ## lambdas self.lbda_adv = 0.1 self.lbda_cons = 1 self.discriminator = self.build_discriminator() self.discriminator.compile(loss=self.custom_discriminator_loss(), optimizer=optimizer, metrics=['mae']) self.generator = self.build_generator(drop_frac = 0) self.generator_u1 = self.build_generator(drop_frac = self.u1) self.generator_u2 = self.build_generator(drop_frac = self.u2) self.generator_u1.trainable = False self.generator_u2.trainable = False z = Input(shape=(self.ld_genes_shape,)) distrib = Input(shape=(self.target_genes_shape,)) target_predicted_u1 = Input(shape=(1,)) target_predicted = self.generator(z) self.discriminator.trainable = False lambda_mult = layers.Lambda(lambda x: tf.multiply(x[0],x[1])) target_predicted_masked = lambda_mult([distrib,target_predicted]) validity_predicted = self.discriminator([z,target_predicted_masked]) concat_out = layers.concatenate([target_predicted,validity_predicted],axis =1) self.combined = Model(inputs=[z,distrib,target_predicted_u1], outputs=concat_out) self.combined.compile(loss=self.custom_generator_loss(), optimizer=optimizer)
def __init__(self, C=4, V=40000, MAX_LEN=600, embed_matrix=None, name='gateddeepcnnmodel.h5', PE=False, train_embed=False): self.MAX_LEN = MAX_LEN self.PE = PE self.name = name input = Input(shape=(MAX_LEN, ), dtype='int32') #CNN不支持mask,即 mask_zero=True if embed_matrix is None: x = Embedding(V, 32)(input) else: embed1 = Embedding(embed_matrix.shape[0], embed_matrix.shape[1], weights=[embed_matrix], trainable=train_embed) x = embed1(input) if self.PE: e_input = Input(shape=(MAX_LEN, ), dtype='int32', name='PE_in') ex = Embedding(self.MAX_LEN, 32, name='PE')(e_input) x = Concatenate()([x, ex]) inputs = x x = Conv1D(128, 2, padding='same', activation='sigmoid')(inputs) xo = Conv1D(128, 2, padding='same')(inputs) inputs = Multiply()([x, xo]) for li in range(3): x = Conv1D(128, 2, padding='same', activation='sigmoid')(inputs) xo = Conv1D(128, 2, padding='same', activation='relu')(inputs) x = Multiply()([x, xo]) inputs = Add()([x, inputs]) # inputs = BatchNormalization()(inputs) # inputs = Activation('relu')(inputs) h = inputs # h = GlobalMaxPool1D()(h) h1 = GlobalMaxPool1D()(h) h2 = GlobalAveragePooling1D()(h) h = Concatenate()([h1, h2]) # hs = BatchNormalization()(hs) z = Dense(128, activation='relu')(h) # z = BatchNormalization()(z) z = Dense(C, activation='softmax')(z) if self.PE: model = Model([input, e_input], z) else: model = Model(input, z) # opt = Adagrad(lr=0.005) # opt = SGDWithWeightnorm(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) opt = AdamWithWeightnorm(lr=0.001) model.compile(opt, 'categorical_crossentropy', metrics=['acc']) self.model = model
def get_optimizer(opt, lr): if opt == "SGD": return SGD(lr=lr) elif opt == "RMSprop": return RMSprop(lr=lr) elif opt == "Adam": return Adam(lr=lr, beta_1=0.5) elif opt == "AdamWithWeightnorm": return AdamWithWeightnorm(lr=lr, beta_1=0.5) elif opt == "SGDWithWeightnorm": return SGDWithWeightnorm(lr=lr)
def train(x_train, y_train, x_test, y_test, maxlen, max_token, embedding_matrix, embedding_dims, batch_size, epochs, logpath, modelpath, modelname, num_classes): embedding_layer = Embedding(max_token + 1, embedding_dims, weights=[embedding_matrix], input_length=maxlen, trainable=True) sentence_input = Input(shape=(maxlen, ), dtype='float64') embedded_sequences = embedding_layer(sentence_input) embedded_sequences = Dropout(0.25)(embedded_sequences) # embed = Embedding(len(vocab) + 1,300, input_length = 20)(inputs) lstm = Bidirectional( LSTM(512, dropout=0.2, recurrent_dropout=0.1, return_sequences=True))(embedded_sequences) attention = AttLayer()(lstm) output = Dense(num_classes, activation='sigmoid')(attention) model = Model(sentence_input, output) model.compile(optimizer=AdamWithWeightnorm(), loss='categorical_crossentropy', metrics=['accuracy']) if os.path.exists('./model490w/GRUAttention-weightnorm-512.h5'): print("Loading the model --GRUAttention-weightnorm-512.h5") model.load_weights('./model490w/GRUAttention-weightnorm-512.h5') early_stopping = EarlyStopping(monitor='acc', patience=10) checkpoint = keras.callbacks.ModelCheckpoint(filepath=modelpath + modelname + '.h5', monitor='acc', verbose=1, save_best_only=True, save_weights_only=True, mode='max', period=1) tensorboard = keras.callbacks.TensorBoard( log_dir='./logw490/{}/'.format(modelname), histogram_freq=0, write_graph=True, write_images=True) callback_lists = [tensorboard, checkpoint, early_stopping] hist = model.fit_generator( generate_arrays_from_file(train_dataset_file, batch_size=batch_size), validation_data=(x_test, to_categorical(y_test, num_classes)), steps_per_epoch=int(4900000 / batch_size), epochs=epochs, callbacks=callback_lists) max_val_acc = max(hist.history['val_acc']) os.rename(modelpath + modelname + '.h5', modelpath + modelname + '(' + str(max_val_acc) + ')' + '.h5') log_format = "%(asctime)s - %(message)s" logging.basicConfig(filename=logpath, level=logging.DEBUG, format=log_format) logging.warning(modelname) for i in range(len(hist.history["acc"])): strlog = str(i + 1) + " Epoch " + "-loss: " + str( hist.history["loss"][i]) + " -acc: " + str( hist.history["acc"][i]) + " -val_loss: " + str( hist.history["val_loss"][i]) + " -val_acc: " + str( hist.history["val_acc"][i]) logging.warning(strlog)
if __name__ == '__main__': numpy.random.seed(0) dsize = 10000 from keras.datasets import mnist (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = X_train.astype(np.float32) X_train = X_train.reshape((X_train.shape[0], -1)) X_train = X_train[:dsize, :] X_test = X_test.astype(np.float32) X_test = X_test.reshape((X_test.shape[0], -1)) X_train /= 255 X_test /= 255 optimizer = AdamWithWeightnorm() model = Sequential() model.add(Dense(1024, input_dim=28 * 28, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(196, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(28 * 28, activation='relu')) model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=[metrics.mean_squared_error]) # nb_epochs in older version cb = TestCallback((X_train, X_train), (X_test, X_test), prefix)
def get_U_Net_model(gpus=1, load_weights=None, verbose=False): from keras.models import Model, load_model from keras.layers import Input, Add, Activation, Dropout from keras.layers.core import Lambda from keras.layers.convolutional import Conv2D, Conv2DTranspose from keras.layers.pooling import MaxPooling2D from keras.layers.merge import concatenate from keras.callbacks import EarlyStopping, ModelCheckpoint from weightnorm import AdamWithWeightnorm from keras.utils import multi_gpu_model from keras import backend as K import tensorflow as tf IMG_WIDTH = conf.U_NET_DIM IMG_HEIGHT= conf.U_NET_DIM IMG_CHANNELS= 3 def conv(f, k=3, act='elu'): return Conv2D(f, (k, k), activation=act, kernel_initializer='he_normal', padding='same') def _incept_conv(inputs, f, chs=[0.15, 0.5, 0.25, 0.1]): fs = [] # determine channel number for k in chs: t = max(int(k*f), 1) # at least 1 channel fs.append(t) fs[1] += f-np.sum(fs) # reminding channels allocate to 3x3 conv c1x1 = conv(fs[0], 1, act='linear') (inputs) c3x3 = conv(max(1, fs[1]//2), 1, act='elu') (inputs) c5x5 = conv(max(1, fs[2]//3), 1, act='elu') (inputs) cpool= MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same') (inputs) c3x3 = conv(fs[1], 3, act='linear') (c3x3) c5x5 = conv(fs[2], 5, act='linear') (c5x5) cpool= conv(fs[3], 1, act='linear') (cpool) output = concatenate([c1x1, c3x3, c5x5, cpool], axis=-1) return output def _res_conv(inputs, f, k=3): # very simple residual module channels = int(inputs.shape[-1]) cs = _incept_conv(inputs, f) if f!=channels: t1 = conv(f, 1, 'linear') (inputs) # identity mapping else: t1 = inputs out = Add()([t1, cs]) # t1 + c2 out = Activation('elu') (out) out = Dropout(0.2) (out) return out def pool(): return MaxPooling2D((2, 2)) def up(inputs): upsampled = Conv2DTranspose(int(inputs.shape[-1]), (2, 2), strides=(2, 2), padding='same') (inputs) return upsampled inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)) c1 = _res_conv(inputs, 32, 3) c1 = _res_conv(c1, 32, 3) p1 = pool() (c1) c2 = _res_conv(p1, 64, 3) c2 = _res_conv(c2, 64, 3) p2 = pool() (c2) c3 = _res_conv(p2, 128, 3) c3 = _res_conv(c3, 128, 3) p3 = pool() (c3) c4 = _res_conv(p3, 256, 3) c4 = _res_conv(c4, 256, 3) p4 = pool() (c4) c5 = _res_conv(p4, 512, 3) c5 = _res_conv(c5, 512, 3) p5 = pool() (c5) c6 = _res_conv(p5, 512, 3) c6 = _res_conv(c6, 512, 3) u7 = up (c6) c7 = concatenate([u7, c5]) c7 = _res_conv(c7, 512, 3) c7 = _res_conv(c7, 512, 3) u8 = up (c7) c8 = concatenate([u8, c4]) c8 = _res_conv(c8, 256, 3) c8 = _res_conv(c8, 256, 3) u9 = up (c8) c9 = concatenate([u9, c3]) c9 = _res_conv(c9, 128, 3) c9 = _res_conv(c9, 128, 3) u10 = up (c9) c10 = concatenate([u10, c2]) c10 = _res_conv(c10, 64, 3) c10 = _res_conv(c10, 64, 3) u11 = up (c10) c11 = concatenate([u11, c1]) c11 = _res_conv(c11, 32, 3) c11 = _res_conv(c11, 32, 3) netout = Conv2D(3, (1, 1), activation='sigmoid', name='nuclei') (c11) optimizer = AdamWithWeightnorm(**conf.U_NET_OPT_ARGS) with tf.device('/cpu:0'): ## prevent OOM error cpu_model = Model(inputs=[inputs], outputs=[netout]) cpu_model.compile(loss=losses.custom_loss, metrics=[metrics.mean_iou, metrics.mean_iou_marker], optimizer=optimizer) if load_weights is not None and os.path.exists(load_weights): cpu_model.load_weights(load_weights) if verbose: print('Loaded weights') if gpus>=2: gpu_model = multi_gpu_model(cpu_model, gpus=gpus) gpu_model.compile(loss=losses.custom_loss, metrics=[metrics.mean_iou, metrics.mean_iou_marker], optimizer=optimizer) return gpu_model, cpu_model return cpu_model, cpu_model
def train(x_train, y_train, x_test, y_test, maxlen, max_token, embedding_matrix, embedding_dims, batch_size, epochs, logpath, modelpath, modelname, num_classes): print(modelname + 'Build model...') sentence = Input(shape=(None, ), dtype='float64') embedding_layer = Embedding(max_token + 1, output_dim=embedding_dims, input_length=maxlen, weights=[embedding_matrix], trainable=True) # print("sentence:",sentence) sentence_embedding = embedding_layer(sentence) drp = Dropout(0.2)(sentence_embedding) block1 = Convolution1D(128, 1, padding='same')(drp) conv2_1 = Convolution1D(256, 1, padding='same')(drp) bn2_1 = BatchNormalization()(conv2_1) relu2_1 = Activation('relu')(bn2_1) block2 = Convolution1D(128, 3, padding='same')(relu2_1) conv3_1 = Convolution1D(256, 3, padding='same')(drp) bn3_1 = BatchNormalization()(conv3_1) relu3_1 = Activation('relu')(bn3_1) block3 = Convolution1D(128, 5, padding='same')(relu3_1) block4 = Convolution1D(128, 3, padding='same')(drp) inception = concatenate([block1, block2, block3, block4], axis=-1) flat = Flatten()(inception) fc = Dense(128)(flat) drop = Dropout(0.5)(fc) relu = Activation('relu')(drop) output = Dense(num_classes, activation='sigmoid')(relu) model = Model(inputs=sentence, outputs=output) model.compile(optimizer=AdamWithWeightnorm(), loss='categorical_crossentropy', metrics=['accuracy']) if os.path.exists('./model490w/TextCNN.h5'): print("loading model TextCNN.h5 ... ") model.load_weights('./model490w/TextCNN.h5') early_stopping = EarlyStopping(monitor='val_acc', patience=3) checkpoint = keras.callbacks.ModelCheckpoint(filepath=modelpath + modelname + '.h5', monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True, mode='max', period=1) tensorboard = keras.callbacks.TensorBoard( log_dir='./log490w/{}/'.format(modelname), histogram_freq=0, write_graph=True, write_images=True) callback_lists = [tensorboard, checkpoint, early_stopping] hist = model.fit_generator( generate_arrays_from_file(train_dataset_file, batch_size=batch_size), validation_data=(x_test, to_categorical(y_test, num_classes)), steps_per_epoch=int(4900000 / batch_size), nb_epoch=epochs, callbacks=callback_lists) max_val_acc = max(hist.history['val_acc']) os.rename(modelpath + modelname + '.h5', modelpath + modelname + '(' + str(max_val_acc) + ')' + '.h5') # print(hist.history) ##输出loss与acc到日志文件 log_format = "%(asctime)s - %(message)s" logging.basicConfig(filename=logpath, level=logging.DEBUG, format=log_format) logging.warning(modelname) for i in range(len(hist.history["acc"])): strlog = str(i + 1) + " Epoch " + "-loss: " + str( hist.history["loss"][i]) + " -acc: " + str( hist.history["acc"][i]) + " -val_loss: " + str( hist.history["val_loss"][i]) + " -val_acc: " + str( hist.history["val_acc"][i]) logging.warning(strlog)
def train2(x_train, y_train, x_test, y_test, maxlen, max_token, embedding_matrix, embedding_dims, batch_size, epochs, logpath, modelpath, modelname, num_classes): embedding_layer = Embedding(max_token + 1, embedding_dims, input_length=maxlen, weights=[embedding_matrix], trainable=True) print(modelname + 'Build model...') model = Sequential() model.add(embedding_layer) model.add(Dropout(0.2)) # model.add(BatchNormalization()) # model.add(SimpleRNN(128, activation="relu")) # model.add(LSTM(128)) model.add(Bidirectional(LSTMpeephole(units=256))) model.add(Dense(num_classes, activation='sigmoid')) ''' from weightnorm import SGDWithWeightnorm sgd_wn = SGDWithWeightnorm(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy',optimizer=sgd_wn,metrics=['accuracy']) ''' # try using different optimizers and different optimizer configs model.compile(optimizer=AdamWithWeightnorm(), loss='categorical_crossentropy', metrics=['accuracy']) model.load_weights( './model490w/TextBiLSTM-weightnorm(0.9156999999237061).h5') # lstm常选参数model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2)) # a stateful LSTM model # lahead: the input sequence length that the LSTM # https://github.com/keras-team/keras/blob/master/examples/lstm_stateful.py # model = Sequential() # model.add(LSTM(20,input_shape=(lahead, 1), # batch_size=batch_size, # stateful=stateful)) # model.add(Dense(num_classes)) # model.compile(loss='mse', optimizer='adam') # patience经过几个epoch后loss不在变化停止训练 early_stopping = EarlyStopping(monitor='val_acc', patience=3) checkpoint = keras.callbacks.ModelCheckpoint(filepath=modelpath + modelname + '.h5', monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=True, mode='max', period=1) tensorboard = keras.callbacks.TensorBoard( log_dir='./log490w/{}/'.format(modelname), histogram_freq=0, write_graph=True, write_images=True) callback_lists = [tensorboard, checkpoint, early_stopping] hist = model.fit_generator( generate_arrays_from_file(train_dataset_file, batch_size=batch_size), validation_data=(x_test, to_categorical(y_test, num_classes)), steps_per_epoch=int(4900000 / batch_size), epochs=epochs, callbacks=callback_lists) max_val_acc = max(hist.history['val_acc']) os.rename(modelpath + modelname + '.h5', modelpath + modelname + '(' + str(max_val_acc) + ')' + '.h5') # print(hist.history) ##输出loss与acc到日志文件 log_format = "%(asctime)s - %(message)s" logging.basicConfig(filename=logpath, level=logging.DEBUG, format=log_format) logging.warning(modelname) for i in range(len(hist.history["acc"])): strlog = str(i + 1) + " Epoch " + "-loss: " + str( hist.history["loss"][i]) + " -acc: " + str( hist.history["acc"][i]) + " -val_loss: " + str( hist.history["val_loss"][i]) + " -val_acc: " + str( hist.history["val_acc"][i]) logging.warning(strlog)
model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) # let's train the model using SGD + momentum (how original). EDIT: now with weight normalization, so slightly more original ;-) from weightnorm import SGDWithWeightnorm from weightnorm import AdamWithWeightnorm sgd_wn = SGDWithWeightnorm(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) adam = optimizers.Adam() adam_wn = AdamWithWeightnorm() if optimizer == 'sgd': optimizer = sgd elif optimizer == 'sgd_wn': optimizer = sgd_wn elif optimizer == 'adam': optimizer = adam elif optimizer == 'adam_wn': optimizer = adam_wn else: assert False model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
class CepstralQSRCNN(object): def __init__(self, opt_params={'lr': 5e-4, 'batch_size': 32, 'nb_epochs': 100}, model_params={'n1': 16, 'n2': 32, 'n3': 16, 'frame_len': 32}, codec_type_params={'codec': 'ADPCM', 'type': '3', 'weights_dir': "./model_weights", 'logdir': "./log"}): self.learning_rate = opt_params['lr'] # Learning rate self.batch_size = opt_params['batch_size'] # Batch size self.nb_epochs = opt_params['nb_epochs'] # Number of epochs self.codec = codec_type_params['codec'] # Codec type self.type = codec_type_params['type'] # Methods type self.log_dir = codec_type_params['logdir'] # Log file direction if not (os.path.exists(self.log_dir)): os.makedirs(self.log_dir) self.weights_dir = codec_type_params['weights_dir'] # Weights file direction if not (os.path.exists(self.weights_dir)): os.makedirs(self.weights_dir) self.frame_len = model_params['frame_len'] # Frame length self.model_params = model_params self.model = self.create_model("qsrcnn") # ------------------------------------------------------------------------------- # Load the Weights of the Model # ------------------------------------------------------------------------------- def load_weights(self, file_path=""): if file_path == "": file_path = self.weights_dir + '/' + self.codec + '_Type' + self.type + '_CepstralQSRCNN_Weights_Best_bs' + \ str(self.batch_size) + '_lr' + str(self.learning_rate) + '.h5' file_path = os.path.normcase(file_path) self.model.load_weights(file_path) # ------------------------------------------------------------------------------- # Save the Weights of the Model # ------------------------------------------------------------------------------- def save_weights(self): file_path = self.weights_dir + '/' + self.codec + '_Type' + self.type + '_CepstralQSRCNN_Weights_Final_bs' + \ str(self.batch_size) + '_lr' + str(self.learning_rate) + '.h5' file_path = os.path.normcase(file_path) self.model.save_weights(file_path) """ def _upscale_block(self, ip, id): init = ip x = Conv1D(256, 3, padding='same', name='espcnn_upconv1_%d' % id)(init) x = Activation(selu)(x) x = SubPixelUpscaling(r=2, channels=64, name='espcnn_upconv1__upscale1_%d' % id)(x) x = Conv1D(256, 3, padding='same', name='espcnn_upconv1_filter1_%d' % id)(x) x = Activation(selu)(x) return x """ # ------------------------------------------------------------------------------- # 1. define model # ------------------------------------------------------------------------------- def create_model(self, model_type="qsrcnn"): if model_type == "qsrcnn": frame_len = self.frame_len n1 = self.model_params['n1'] n2 = self.model_params['n2'] n3 = self.model_params['n3'] input_sque = Input(shape=(frame_len, 1)) c1 = Conv1D(n1, 3, padding='same')(input_sque) c1 = Activation(selu)(c1) c1 = Conv1D(n1, 3, padding='same')(c1) c1 = Activation(selu)(c1) x = MaxPooling1D(2)(c1) c2 = Conv1D(n2, 3, padding='same')(x) c2 = Activation(selu)(c2) c2 = Conv1D(n2, 3, padding='same')(c2) c2 = Activation(selu)(c2) x = MaxPooling1D(2)(c2) c3 = Conv1D(n3, 3, padding='same')(x) c3 = Activation(selu)(c3) x = UpSampling1D(2)(c3) c2_2 = Conv1D(n2, 3, padding='same')(x) c2_2 = Activation(selu)(c2_2) c2_2 = Conv1D(n2, 3, padding='same')(c2_2) c2_2 = Activation(selu)(c2_2) m1 = Add()([c2, c2_2]) m1 = UpSampling1D(2)(m1) c1_2 = Conv1D(n1, 3, padding='same')(m1) c1_2 = Activation(selu)(c1_2) c1_2 = Conv1D(n1, 3, padding='same')(c1_2) c1_2 = Activation(selu)(c1_2) m2 = Add()([c1, c1_2]) decoded = Conv1D(1, 5, padding='same', activation='linear')(m2) model = Model(input_sque, decoded) elif model_type == "wavenet": frame_len = self.frame_len ae_width = 16 ae_filter_length = 3 ae_num_stages = 2 ae_num_layers = 6 num_stages = 2 num_layers = 6 width = 16 skip_width = 16 filter_length = 3 input_sque = Input(shape=(frame_len, 1), name='input_layer') # --------------------------------------- # The Non-Causal Temporal Encoder. # --------------------------------------- en = Conv1D(ae_width, ae_filter_length, padding='same', name='ae_startconv')(input_sque) for num_layer in range(ae_num_layers): # dilation: 2**(0 1 2 3 4) d = Activation(selu)(en) d = Conv1D(ae_width, 3, padding='same', dilation_rate=2 ** (num_layer % ae_num_stages), name='ae_dilatedconv_%d' % (num_layer + 1))(d) d = Activation(selu)(d) en2 = Conv1D(ae_width, 1, padding='same', dilation_rate=2 ** (num_layer % ae_num_stages), name='ae_res_%d' % (num_layer + 1))(d) en = Add()([en2, en]) en = Activation(selu)(en) en = Conv1D(16, 1, padding='causal', dilation_rate=1, name='ae_bottleneck')(en) en = Activation(selu)(en) en = AveragePooling1D(2, name='ae_pool')(en) # encoding = en # --------------------------------------- # The WaveNet Decoder. # --------------------------------------- # enup = UpSampling1D(2, name='up_sampling')(en) # l = shift_right(input_frame) l = Conv1D(width, filter_length, padding='causal', dilation_rate=1, name='startconv')(input_sque) l = Activation(selu)(l) # Set up skip connections. s = Conv1D(skip_width, 1, padding='causal', dilation_rate=1, name='skip_start')(l) s = Activation(selu)(s) # Residual blocks with skip connections. for i in range(num_layers): d = Conv1D(2 * width, filter_length, padding='causal', dilation_rate=2 ** (i % num_stages), name='dilatedconv_%d' % (i + 1))(l) d = Activation(selu)(d) en3 = Conv1D(2 * width, 1, padding='causal', dilation_rate=1, name='cond_map_%d' % (i + 1))(en) # 40 en3 = Activation(selu)(en3) en3 = UpSampling1D(2, name='up_sampling_%d' % (i + 1))(en3) # d = condition(d,en3) d = Add()([d, en3]) d_sigmoid = Activation('sigmoid')(d) d_tanh = Activation('tanh')(d) d = Multiply()([d_sigmoid, d_tanh]) l2 = Conv1D(width, 1, padding='causal', dilation_rate=1, name='res_%d' % (i + 1))(d) l2 = Activation(selu)(l2) l = Add()([l2, l]) s2 = Conv1D(skip_width, 1, padding='causal', dilation_rate=1, name='skip_%d' % (i + 1))(d) s = Add()([s2, s]) s = Activation(selu)(s) s = Conv1D(skip_width, 3, padding='causal', activation='linear', name='output_layer1')(s) s = Activation(selu)(s) en4 = Conv1D(skip_width, 1, padding='causal', activation='linear', name='cond_map_out1')(en) en4 = Activation(selu)(en4) en4 = UpSampling1D(2, name='up_sampling')(en4) s = Add()([en4, s]) s = Activation(selu)(s) outs = Conv1D(1, 3, padding='causal', activation='linear', name='output_layer')(s) model = Model(input_sque, outs) elif model_type == "autoencoder": frame_len = self.frame_len n1 = 64 n2 = 32 input_sque = Input(shape=(frame_len, 1)) c1 = Conv1D(n1, 3, padding='same')(input_sque) c1 = Activation(selu)(c1) x = MaxPooling1D(2)(c1) c2 = Conv1D(n2, 3, padding='same')(x) c2 = Activation(selu)(c2) encoded = MaxPooling1D(2)(c2) d1 = UpSampling1D(2)(encoded) d1 = Conv1D(n2, 3, padding='same')(d1) d1 = Activation(selu)(d1) y = Activation(selu)(d1) d2 = UpSampling1D(2)(y) d2 = Conv1D(n1, 3, padding='same')(d2) d2 = Activation(selu)(d2) decoded = Conv1D(1, 5, padding='same', activation='linear')(d2) model = Model(input_sque, decoded) elif model_type == "esrcnn": f1 = 5 f2_1 = 1 f2_2 = 2 f2_3 = 3 f3 = 5 n1 = 128 n2 = 64 frame_len = self.frame_len input_img = Input(shape=(frame_len, 1)) x = Conv1D(n1, f1, padding='same', name='level1')(input_img) x = Activation(selu)(x) x1 = Conv1D(n2, f2_1, padding='same', name='lavel1_1')(x) x1 = Activation(selu)(x1) x2 = Conv1D(n2, f2_2, padding='same', name='lavel1_2')(x) x2 = Activation(selu)(x2) x3 = Conv1D(n2, f2_3, padding='same', name='lavel1_3')(x) x3 = Activation(selu)(x3) x = Average()([x1, x2, x3]) out = Conv1D(1, f3, padding='same', activation='linear', name='output_1')(x) # out = LeakyReLU(0.2)(out) model = Model(input_img, out) """ elif model_type == "subpixel": frame_len = self.frame_len input_frame = Input(shape=(frame_len, 1)) x = Conv1D(64, 5, padding='same', name='level1')(input_frame) x = Activation(selu)(x) x = Conv1D(32, 3, padding='same', name='level2')(x) x = Activation(selu)(x) x = self._upscale_block(x, 1) out = Conv1D(1, 5, activation='linear', padding='same', name='output_1')(x) model = Model(input_frame, out) """ model.summary() learning_rate = self.learning_rate # adam = optimizers.Adam(lr=learning_rate) # model.compile(optimizer=adam, loss='mse', metrics=[SNRLoss]) adam_wn = AdamWithWeightnorm(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(optimizer=adam_wn, loss='mse', metrics=[snr]) return model
X_train /= 255 X_test /= 255 input_dim = 32 * 32 * 3 model = Sequential() model.add(Dense(1024, input_dim=input_dim, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(196, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(1024, activation='relu')) model.add(Dense(input_dim, activation='relu')) #sgd_wn = SGDWithWeightnorm(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) sgd_wn = AdamWithWeightnorm() model.compile(loss='mean_squared_error', optimizer=sgd_wn, metrics=[metrics.mean_squared_error]) cb = TestCallback((X_train, X_train), (X_test, X_test), prefix) # data based initialization of parameters from weightnorm import data_based_init data_based_init(model, X_train[:100]) model.fit(X_train, X_train, batch_size=batch_size, epochs=nb_epoch, validation_data=(X_test, X_test), callbacks=[cb],
model.add(Activation('relu')) model.add(Convolution2D(64, 3, 3)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) # let's train the model using SGD + momentum (how original). EDIT: now with weight normalization, so slightly more original ;-) from weightnorm import AdamWithWeightnorm adam = AdamWithWeightnorm(lr=0.001, decay=1e-6) #adam = Adam(lr=0.01) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) # data based initialization of parameters from weightnorm import data_based_init data_based_init(model, X_train[:100]) if not data_augmentation: print('Not using data augmentation.') model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
def wgangp_conditional(h=128, w=128, c=3, latent_dim=2, condition_dim=10, epsilon_std=1.0, dropout_rate=0.1, GRADIENT_PENALTY_WEIGHT=10): optimizer_g = AdamWithWeightnorm(lr=0.0001, beta_1=0.5) optimizer_d = AdamWithWeightnorm(lr=0.0001, beta_1=0.5) optimizer_c = AdamWithWeightnorm(lr=0.0001, beta_1=0.5) t_h, t_w = h // 16, w // 16 generator = residual_decoder(t_h, t_w, c=c, latent_dim=latent_dim + condition_dim, dropout_rate=dropout_rate) discriminator = residual_discriminator(h=h, w=w, c=c, dropout_rate=dropout_rate, return_hidden=True) classifier = residual_discriminator(h=h, w=w, c=c, dropout_rate=dropout_rate, as_classifier=condition_dim, return_hidden=True) for layer in discriminator.layers: layer.trainable = False discriminator.trainable = False for layer in classifier.layers: layer.trainable = False classifier.trainable = False generator_input = Input(shape=(latent_dim + condition_dim, )) generator_layers = generator(generator_input) discriminator_layers_for_generator = discriminator(generator_layers)[0] classifier_layers_for_generator = classifier(generator_layers)[0] generator_model = Model(inputs=[generator_input], outputs=[ discriminator_layers_for_generator, classifier_layers_for_generator ]) generator_model.add_loss(K.mean(discriminator_layers_for_generator)) generator_model.compile(optimizer=optimizer_g, loss=[None, 'categorical_crossentropy']) # Now that the generator_model is compiled, we can make the discriminator layers trainable. for layer in discriminator.layers: layer.trainable = True for layer in generator.layers: layer.trainable = False discriminator.trainable = True generator.trainable = False # The discriminator_model is more complex. It takes both real image samples and random noise seeds as input. # The noise seed is run through the generator model to get generated images. Both real and generated images # are then run through the discriminator. Although we could concatenate the real and generated images into a # single tensor, we don't (see model compilation for why). real_samples = Input(shape=(h, w, c)) generator_input_for_discriminator = Input(shape=(latent_dim + condition_dim, )) generated_samples_for_discriminator = generator( generator_input_for_discriminator) discriminator_output_from_generator = discriminator( generated_samples_for_discriminator)[0] discriminator_output_from_real_samples, d0, d1, d2 = discriminator( real_samples) classifier_output_from_real_samples, c0, c1, c2 = classifier(real_samples) ds = K.concatenate([K.flatten(d0), K.flatten(d1), K.flatten(d2)], axis=-1) cs = K.concatenate([K.flatten(c0), K.flatten(c1), K.flatten(c2)], axis=-1) c_loss = .1 * K.mean(K.square(ds - cs)) averaged_samples = RandomWeightedAverage()( [real_samples, generated_samples_for_discriminator]) averaged_samples_out = discriminator(averaged_samples)[0] discriminator_model = Model( [real_samples, generator_input_for_discriminator], [ discriminator_output_from_real_samples, discriminator_output_from_generator, averaged_samples_out ]) discriminator_model.add_loss( K.mean(discriminator_output_from_real_samples) - K.mean(discriminator_output_from_generator) + gradient_penalty_loss( averaged_samples_out, averaged_samples, GRADIENT_PENALTY_WEIGHT)) discriminator_model.add_loss(c_loss, inputs=[discriminator]) discriminator_model.compile(optimizer=optimizer_d, loss=None) for layer in classifier.layers: layer.trainable = True classifier.trainable = True classifier_model = Model([real_samples], [classifier_output_from_real_samples]) classifier_model.add_loss(c_loss, inputs=[classifier]) classifier_model.compile(optimizer=optimizer_c, loss='categorical_crossentropy') return generator_model, discriminator_model, classifier_model, generator, discriminator, classifier
def build_gan(h=128, w=128, c=3, latent_dim=2, epsilon_std=1.0, dropout_rate=0.1, GRADIENT_PENALTY_WEIGHT=10): optimizer_g = AdamWithWeightnorm(lr=0.0001, beta_1=0.5) optimizer_d = AdamWithWeightnorm(lr=0.0001, beta_1=0.5) t_h, t_w = h // 16, w // 16 generator = residual_decoder(t_h, t_w, c=c, latent_dim=latent_dim, dropout_rate=dropout_rate) discriminator = residual_discriminator(h=h, w=w, c=c, dropout_rate=dropout_rate) for layer in discriminator.layers: layer.trainable = False discriminator.trainable = False generator_input = Input(shape=(latent_dim, )) generator_layers = generator(generator_input) discriminator_layers_for_generator = discriminator(generator_layers) generator_model = Model(inputs=[generator_input], outputs=[discriminator_layers_for_generator]) generator_model.add_loss(K.mean(discriminator_layers_for_generator)) generator_model.compile(optimizer=optimizer_g, loss=None) # Now that the generator_model is compiled, we can make the discriminator layers trainable. for layer in discriminator.layers: layer.trainable = True for layer in generator.layers: layer.trainable = False discriminator.trainable = True generator.trainable = False # The discriminator_model is more complex. It takes both real image samples and random noise seeds as input. # The noise seed is run through the generator model to get generated images. Both real and generated images # are then run through the discriminator. Although we could concatenate the real and generated images into a # single tensor, we don't (see model compilation for why). real_samples = Input(shape=(h, w, c)) generator_input_for_discriminator = Input(shape=(latent_dim, )) generated_samples_for_discriminator = generator( generator_input_for_discriminator) discriminator_output_from_generator = discriminator( generated_samples_for_discriminator) discriminator_output_from_real_samples = discriminator(real_samples) averaged_samples = RandomWeightedAverage()( [real_samples, generated_samples_for_discriminator]) averaged_samples_out = discriminator(averaged_samples) discriminator_model = Model( [real_samples, generator_input_for_discriminator], [ discriminator_output_from_real_samples, discriminator_output_from_generator, averaged_samples_out ]) discriminator_model.add_loss( K.mean(discriminator_output_from_real_samples) - K.mean(discriminator_output_from_generator) + gradient_penalty_loss( averaged_samples_out, averaged_samples, GRADIENT_PENALTY_WEIGHT)) discriminator_model.compile(optimizer=optimizer_d, loss=None) return generator_model, discriminator_model, generator, discriminator