def fit(self, X, y, X_t, y_t): X = self.transfer_shape(X) X_t = self.transfer_shape(X_t) y = y[2-1:] y_t = y_t[2-1:] #self.classifier.add(Dense(500, input_shape=( X.shape[1],))) self.classifier.add(LSTM(input_shape=(2, X.shape[2]), output_dim =8, return_sequences = True, kernel_initializer=initializers.glorot_normal(123) )) self.classifier.add(Flatten()) self.classifier.add(Activation('relu')) self.classifier.add(Dropout(0.3, seed=1234)) self.classifier.add(Dense( output_dim=4, kernel_initializer=initializers.glorot_normal(123))) self.classifier.add(Activation('relu')) self.classifier.add(Dropout(0.3, seed=123)) self.classifier.add(Dense( output_dim=4, kernel_initializer=initializers.glorot_normal(123))) self.classifier.add(Activation('relu')) self.classifier.add(Dropout(0.3, seed=123)) self.classifier.add(Dense(output_dim=4)) self.classifier.add(Activation('tanh')) #self.classifier.add(Activation('relu')) self.classifier.add(Dense(output_dim=1, kernel_initializer=initializers.glorot_normal(123))) self.classifier.add(Activation('sigmoid')) sgd = SGD(lr=0.01) opt = Adam(lr=4e-5) opt = Adam() #opt = RMSprop(lr=4e-3) #opt = Adadelta() self.classifier.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) self.classifier.fit(X, y, validation_data=(X_t, y_t), batch_size=self.batch_size, nb_epoch=self.nb_epoch)
def Model1(embedding_matrix=None): embedding_layer = Embedding(len(word_index) + 1, EMBEDDING_DIM, weights=embedding_matrix, input_length=MAX_SEQUENCE_LENGTH, trainable=True) sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) net = SpatialDropout1D(0.2)(embedded_sequences) net = Bidirectional(layer=GRU(EMBEDDING_DIM, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_gru')(net) # net = Bidirectional( # layer=LSTM(EMBEDDING_DIM, return_sequences=True, # kernel_initializer=glorot_normal(seed=1029), # recurrent_initializer=orthogonal(gain=1.0, seed=1029)), # name='bidirectional_lstm')(net) #net = BatchNormalization()(net) capsul = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(net) # noqa capsul = Flatten()(capsul) capsul = DropConnect(Dense(8, activation="relu"), prob=0.01)(capsul) atten = Attention(step_dim=MAX_SEQUENCE_LENGTH, name='attention')(net) atten = DropConnect(Dense(4, activation="relu"), prob=0.2)(atten) net = Concatenate(axis=-1)([capsul, atten]) # net = GlobalAveragePooling1D()(net) # output = Dense(units=1, activation='sigmoid', name='output')(net) # net = GlobalAveragePooling1D()(net) net = Dense(100, activation='relu')(net) # net = Dropout(0.3)(net) output = Dense(2, activation='softmax')(net) model2 = Model(inputs=sequence_input, outputs=output) model2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) model2.summary() # Keeping a checkpoint to store only the model which gives best output validation accuracy chkpt2 = ModelCheckpoint('expertiza_nn_model2.h5', monitor='val_acc', verbose=1, save_best_only=True) return (model2, chkpt2)
def create_sharedLSTMmodel(self): inputs = Input(shape=(self.maxlen, self.height, self.width, 3)) x = TimeDistributed(self.resnet, name="resnet")(inputs) x = TimeDistributed(GlobalAveragePooling2D(), name="GAP")(x) x = TimeDistributed(Dense(512, activation='relu'), name="dense")(x) predictions = Bidirectional( LSTM(128, batch_input_shape=(None, self.maxlen, 512), kernel_initializer=glorot_normal(seed=20181020), recurrent_initializer=orthogonal(gain=1.0, seed=20181020), dropout=0.01, recurrent_dropout=0.01))(x) predictions = Reshape((1, 256))(predictions) shared_layers = Model(inputs, predictions, name="shared_LSTMlayers") return shared_layers
def __init__(self, lambda_initializer=initializers.Constant(value=1), t_initializer=initializers.glorot_normal(), shared_axes=None, **kwargs): super(BHSA, self).__init__(**kwargs) self.lambda_initializer = initializers.get(lambda_initializer) self.t_initializer = initializers.get(t_initializer) if (shared_axes == None): self.shared_axes = None elif not isinstance(shared_axes, (list, tuple)): self.shared_axes = [shared_axes] else: self.shared_axes = list(shared_axes)
def CNN___itworks(input_shape, n_class, CNNkernel, CNNChannel, DenseChannel): # ex_name: 0715 cnn_init = initializers.glorot_normal() model = Sequential() model.add(Conv2D(32, kernel_size=(19,19), strides=(1,1), activation='relu', input_shape=input_shape, kernel_initializer=cnn_init, padding='valid')) #model.add(MaxPooling2D(pool_size=(1, 3), strides=(1, 3))) model.add(Conv2D(32, (19,19), activation='relu', kernel_initializer=cnn_init, padding='valid')) #model.add(Conv2D(32, (5,5), activation='relu',kernel_initializer=cnn_init, padding='valid')) model.add(Flatten()) model.add(Dense(256, activation='relu')) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(n_class, activation='softmax')) return model
def FireModule(s_1x1, e_1x1, e_3x3, name): """FireModule Fire module for the SqueezeNet model. Implements the expand layer, which has a mix of 1x1 and 3x3 filters, by using two conv layers concatenated in the channel dimension. :param s_1x1: Number of 1x1 filters in the squeeze layer :param e_1x1: Number of 1x1 filters in the expand layer :param e_3x3: Number of 3x3 filters in the expand layer :param name: Name of the fire module :return: Returns a callable function """ # Concat on the channel axis. TensorFlow uses (rows, cols, channels), while concat_axis = 3 initializer = initializers.glorot_normal() def layer(x): squeeze = Convolution2D(s_1x1, kernel_size=(1, 1), activation='relu', kernel_initializer=initializer, name=name + '/squeeze1x1')(x) squeeze = BatchNormalization(name=name + '/squeeze1x1_bn')(squeeze) # Needed to merge layers expand_1x1 and expand_3x3. expand_1x1 = Convolution2D(e_1x1, kernel_size=(1, 1), activation='relu', kernel_initializer=initializer, name=name + '/expand1x1')(squeeze) # Pad the border with zeros. Not needed as padding='same' will do the same. # expand_3x3 = ZeroPadding2D(padding=(1, 1), name=name+'_expand_3x3_padded')(squeeze) expand_3x3 = Convolution2D(e_3x3, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer=initializer, name=name + '/expand3x3')(squeeze) # Concat in the channel dim expand_merge = concatenate([expand_1x1, expand_3x3], axis=concat_axis, name=name + '/concat') return expand_merge return layer
def ref_crnn(input_shape, n_class, model_size_info): cprint('**** CRNN ****', 'green') assert (len(model_size_info) == 9) cnn_info = model_size_info[:5] rnn_info = model_size_info[5:8] fc_unit = model_size_info[8] init = initializers.glorot_normal() # MODEL model = Sequential() model.add( Conv2D(cnn_info[0], kernel_size=(cnn_info[1], cnn_info[2]), strides=(cnn_info[3], cnn_info[4]), activation='relu', input_shape=input_shape, kernel_initializer=init, padding='valid')) model.add(layers.TimeDistributed(Flatten())) for i in range(rnn_info[0] - 1): if rnn_info[2] == 0: model.add( layers.Bidirectional( layers.LSTM(rnn_info[1], return_sequences=True))) elif rnn_info[2] == 1: model.add( layers.Bidirectional( layers.GRU(rnn_info[1], return_sequences=True))) else: raise ValueError('wrong type name') if rnn_info[2] == 0: model.add(layers.Bidirectional(layers.LSTM(rnn_info[1]))) elif rnn_info[2] == 1: model.add(layers.Bidirectional(layers.GRU(rnn_info[1]))) else: raise ValueError('wrong type name') model.add( Dense(fc_unit, activation='relu', kernel_initializer=init, bias_initializer='zeros')) model.add( Dense(n_class, activation='softmax', kernel_initializer=init, bias_initializer='zeros')) return model
def CNN(input_shape, n_class, CNNkernel, CNNChannel, DenseChannel): # ex_name: 0715 cprint(str(CNNkernel)+str(CNNChannel)+str(DenseChannel),'yellow') cnn_init = initializers.glorot_normal() model = Sequential() model.add(Conv2D(CNNChannel, kernel_size=(CNNkernel,CNNkernel), strides=(1,1), activation='relu', input_shape=input_shape, kernel_initializer=cnn_init, padding='valid')) model.add(MaxPooling2D(pool_size=(1, 3), strides=(1, 3))) model.add(Conv2D(CNNChannel, (CNNkernel,CNNkernel), activation='relu', kernel_initializer=cnn_init, padding='valid')) model.add(Conv2D(CNNChannel, (CNNkernel,CNNkernel), activation='relu',kernel_initializer=cnn_init, padding='valid')) model.add(Flatten()) model.add(Dense(DenseChannel, activation='relu')) model.add(Dense(DenseChannel//2, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(n_class, activation='softmax')) return model
def _make_model(self): step = 0 print('******************************************', step) step += 1 model_to_make = Sequential() print('******************************************', step) step += 1 model_to_make.add( Conv2D(32, (5, 5), kernel_initializer=glorot_normal(), bias_initializer='zeros', strides=(3, 3), data_format='channels_first', input_shape=(3, 23, 23))) print(model_to_make.input_shape) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add(Activation('relu')) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), data_format='channels_first', input_shape=(32, 7, 7))) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add(Activation('relu')) print(model_to_make.output) print('******************************************', 'flattened') model_to_make.add(Flatten()) print(model_to_make.output) model_to_make.add(Dense(units=2, input_dim=288)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Activation('softmax')) print('******************************************', step) step += 1 print('model waiting to be compiled') self.model = model_to_make print('******************************************')
def get_discriminator(input_size, G, G_input_size): """Returns the discriminator networks `D` and `DG`. `input_size` is the input size of `D` and `DG`. `G` is the generator model. Return a tuple of 2 elements: * The first element is `D`, the discriminator/critic * The second element id `DG` -> D(G(z))""" xavier = initializers.glorot_normal() x = Input(shape=(input_size, ), name='input_x') a = Reshape((28, 28, 1))(x) a = Conv2D(16, (3, 3), padding='same', kernel_initializer=xavier)(a) a = Flatten()(a) a = Dense(256, kernel_initializer=xavier)(x) a = LeakyReLU()(a) a = Dropout(0.5)(a) a = Dense(256, kernel_initializer=xavier)(a) a = LeakyReLU()(a) a = Dropout(0.5)(a) # creates an output to determine if an input is fake is_fake = Dense(1, activation='linear', name='output_is_fake')(a) # add the "fork" to classify classes = Dense(10, activation='softmax', name='D_classes')(a) # creates a D model that receives a real example as input D = Model(inputs=[x], outputs=[is_fake, classes], name='D') # creates another model that uses G as input z = Input(shape=(G_input_size, ), name='D_input_z') is_fake, classes = D(G(inputs=[z])) DG = Model(inputs=[z], outputs=[is_fake, classes], name='DG') # D shouldn't be trained during the generator's training faze DG.get_layer('D').trainable = False DG.compile(optimizer=Nadam(lr=0.0002), loss=[mean, 'categorical_crossentropy']) D.trainable = True D.compile(optimizer=Nadam(lr=0.0002), loss=[mean, 'categorical_crossentropy']) return D, DG
def get_generator(input_size, output_size): """Returns the generator model `G`. `input_size` and `output_size` are the size of the input vectors z for `G` and the output of `G`. """ xavier = initializers.glorot_normal() z = Input(shape=(input_size, ), name='input_z') a = Dense(256, kernel_initializer=xavier)(z) a = LeakyReLU()(a) a = Reshape((16, 16, 1))(a) a = Conv2DTranspose(16, (3, 3), kernel_initializer=xavier)(a) a = LeakyReLU()(a) a = Flatten()(a) a = Dense(output_size, activation='sigmoid', kernel_initializer=xavier)(a) G = Model(inputs=[z], outputs=[a], name='G') return G
def jsbae_0318_C2F1(input_shape, n_class): #jsbae_0314_clean_clean ## BEST MODEL UNTIL NOW cnn_init = initializers.glorot_normal() model = Sequential() model.add(Conv2D(128, kernel_size=5, strides=(1,1), activation='relu', input_shape=input_shape, kernel_initializer=cnn_init, padding='valid')) model.add(MaxPooling2D(pool_size=(1, 3), strides=(1, 3))) #model.add(Conv2D(128, (5,5), activation='relu')) model.add(Conv2D(256, (5,5), activation='relu', kernel_initializer=cnn_init, padding='valid')) #model.add(Conv2D(256, (5,5), activation='relu',kernel_initializer=cnn_init, padding='valid')) model.add(Flatten()) model.add(Dense(1024, activation='relu')) #model.add(Dense(1024, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(n_class, activation='softmax')) return model
def train_model(): # some declared variables randomSeed = 42 networkInitialize = glorot_normal() inputImageShape = (224, 224, 3) epoch = 200 btachSize = 32 num_of_output_classes = 2 random.seed(randomSeed) learningRate = 0.01 trainX, testX, trainY, testY = load_data() # augmentation process augmentaion = ImageDataGenerator(rotation_range=30, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode="nearest") checkpoint = ModelCheckpoint( 'models\\model-{epoch:03d}-{acc:03f}-{val_acc:03f}.h5', verbose=1, monitor='val_acc', save_best_only=True, mode='auto') csv_logger = CSVLogger('report\\log_' + str(learningRate) + '.csv', append=False, separator=';') # training # compile the model model = cnn_model_structure(input_shape=inputImageShape, num_classes=num_of_output_classes) model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy']) # print(model.summary()) model = model.fit_generator(augmentaion.flow(trainX, trainY, batch_size=btachSize), validation_data=(testX, testY), steps_per_epoch=len(trainX), epochs=epoch, callbacks=[csv_logger, checkpoint])
def training_data(tmpdir_factory): import h5py from keras.layers import Dense from keras.models import Sequential from keras.optimizers import SGD from keras.initializers import glorot_normal, normal from deepreplay.datasets.parabola import load_data from deepreplay.callbacks import ReplayData filename = str(tmpdir_factory.mktemp('data').join('training.h5')) X, y = load_data(xlim=(-1, 1), n_points=1000, shuffle=True, seed=13) sgd = SGD(lr=0.05) glorot_initializer = glorot_normal(seed=42) normal_initializer = normal(seed=42) replaydata = ReplayData(X, y, filename=filename, group_name='part1_activation_functions') np.random.seed(13) model = Sequential() model.add( Dense(input_dim=2, units=2, kernel_initializer=glorot_initializer, activation='sigmoid', name='hidden')) model.add( Dense(units=1, kernel_initializer=normal_initializer, activation='sigmoid', name='output')) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['acc']) model.fit(X, y, epochs=20, batch_size=16, callbacks=[replaydata]) training_data = h5py.File(filename, 'r') return training_data['part1_activation_functions']
def _make_model(self): step = 0 print( '******************************************', step ) step += 1 model_to_make = Sequential() print( '******************************************', step ) step += 1 model_to_make.add( Conv2D( 32, (5, 5), kernel_initializer=glorot_normal(), bias_initializer='zeros', strides=(3, 3), data_format='channels_first', input_shape=(3, 23, 23) ) ) print( model_to_make.input_shape ) print( model_to_make.output ) print( '******************************************', step ) step += 1 model_to_make.add( Activation( 'relu' ) ) print( model_to_make.output ) print( '******************************************', step ) step += 1 model_to_make.add( Conv2D( filters=32, kernel_size=(3, 3), strides=(2, 2), data_format='channels_first', input_shape=(32, 7, 7) ) ) print( model_to_make.output ) print( '******************************************', step ) step += 1 model_to_make.add( Activation( 'relu' ) ) print( model_to_make.output ) print( '******************************************', 'flattened' ) model_to_make.add( Flatten() ) print( model_to_make.output ) model_to_make.add( Dense( units=2, input_dim=288 ) ) print( '******************************************', step ) step += 1 print( model_to_make.output ) model_to_make.add( Activation( 'softmax' ) ) print( '******************************************', step ) step += 1 print( 'model waiting to be compiled' ) self.model = model_to_make print( '******************************************' )
def agent_init(): optimizer_map = { 'Adam': Adam(lr=a_globs.ALPHA), 'RMSprop': RMSprop(lr=a_globs.ALPHA), 'Adagrad': Adagrad(lr=a_globs.ALPHA), 'SGD': SGD(lr=a_globs.ALPHA) } initializer_map = { 'random': random_uniform(), 'glorot': glorot_normal(), 'he': he_normal() } a_globs.cur_epsilon = a_globs.EPSILON #The main buffer contains all of the sub buffers used to store different types of states, to support biased sampling a_globs.generic_buffer = [] a_globs.buffer_container = [a_globs.generic_buffer] #Initialize the neural network a_globs.model = Sequential() init_weights = initializer_map[a_globs.INIT] a_globs.model.add( Dense(a_globs.NUM_NERONS_LAYER_1, activation='relu', kernel_initializer=init_weights, input_shape=(a_globs.FEATURE_VECTOR_SIZE, ))) a_globs.model.add( Dense(a_globs.NUM_NERONS_LAYER_2, activation='relu', kernel_initializer=init_weights)) a_globs.model.add( Dense(a_globs.NUM_ACTIONS, activation='linear', kernel_initializer=init_weights)) a_globs.model.compile(loss='mse', optimizer=optimizer_map[a_globs.OPTIMIZER]) summarize_model(a_globs.model, a_globs.AGENT) #Create the target network a_globs.target_network = clone_model(a_globs.model) a_globs.target_network.set_weights(a_globs.model.get_weights())
def built_and_compile(params, num_classes): layers = [ Conv1D(params['conv1'], kernel_size=params['kernel_size1'], activation=params['activation1'], input_shape=(params['data_dim'], 1), use_bias=False, kernel_initializer=glorot_normal(seed=7)), BatchNormalization(), MaxPooling1D(params['pool1']), Dropout(rate=params['drop_rate1']), Conv1D(params['conv2'], kernel_size=params['kernel_size2'], activation=params['activation2'], use_bias=False, kernel_initializer=glorot_normal(seed=7)), BatchNormalization(), MaxPooling1D(params['pool2']), Dropout(rate=params['drop_rate2']), Conv1D(params['conv3'], kernel_size=params['kernel_size3'], activation=params['activation3'], use_bias=False, kernel_initializer=glorot_normal(seed=7)), BatchNormalization(), MaxPooling1D(params['pool3']), Dropout(rate=params['drop_rate3']), Conv1D(params['conv4'], kernel_size=params['kernel_size4'], activation=params['activation4'], use_bias=False, kernel_initializer=glorot_normal(seed=7)), BatchNormalization(), MaxPooling1D(params['pool4']), GlobalAveragePooling1D(), Dense(params['dense1'], activation=params['dense1_act'], use_bias=False, kernel_initializer=glorot_normal(seed=7)), BatchNormalization(), Dropout(rate=params['drop_rate4']), Dense(num_classes, activation='softmax', kernel_initializer=glorot_normal(seed=7)) ] model = Sequential(layers) model.compile(loss='categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy']) return model
def _get_init(self): """ ------------------------------------------------------------------------------------------------- Return an Initializer according to the object attribute return: [keras.initializers.Initializer] ------------------------------------------------------------------------------------------------- """ if self.k_initializer == 'RUNIF': return initializers.RandomUniform(minval=-0.05, maxval=0.05, seed=cnfg['seed']) if self.k_initializer == 'GLOROT': return initializers.glorot_normal(seed=cnfg['seed']) if self.k_initializer == 'HE': return initializers.he_normal(seed=cnfg['seed']) ms.print_err("Initializer {} not valid".format(self.k_initializer))
def __init__(self,sess,dim_action,dim_state,n_plan_data,\ n_traj,time_steps,\ save_folder, key_configs=None,x_scaler=None,c_scaler=None): self.initializer = initializers.glorot_normal() self.sess = sess self.key_configs = key_configs self.save_folder = save_folder self.s_scaler = x_scaler self.n_traj = n_traj self.time_steps = time_steps self.n_plan_data = n_plan_data self.noise_term_var = 0.25 #y= x+L * z, Z~N(0,1), then x ~ N(x,L^2)? self.setup_data_dimensions(dim_action, dim_state) self.setup_inputs() self.create_policy()
def create_sharedAutoRegLSTMmodel(self): inputs = Input(shape=(2 * self.maxlen, 1)) mid_lstm = Bidirectional( LSTM(128, batch_input_shape=(None, 2 * self.maxlen, 1), kernel_initializer=glorot_normal(seed=20181020), recurrent_initializer=orthogonal(gain=1.0, seed=20181020), dropout=0.01, recurrent_dropout=0.01, return_sequences=True))(inputs) mid_dense = Dense(256, activation='relu')(mid_lstm) predictions = Dense(self.maxlen, activation='sigmoid')(mid_dense) shared_layers = Model(inputs, predictions, name="shared_AutoRegLSTMlayers") return shared_layers
def get_model(embed_weights): input_layer = Input(shape=(MAX_LEN, ), name='input') # 1. embedding layer # get embedding weights print('load pre-trained embedding weights ......') input_dim = embed_weights.shape[0] output_dim = embed_weights.shape[1] x = Embedding(input_dim=input_dim, output_dim=output_dim, weights=[embed_weights], trainable=False, name='embedding')(input_layer) # clean up del embed_weights, input_dim, output_dim gc.collect() # 2. dropout x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x) # 3. bidirectional lstm x = Bidirectional(layer=CuDNNLSTM( RNN_UNITS, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm')(x) # 4. capsule layer capsul = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) # noqa capsul = Flatten()(capsul) capsul = DropConnect(Dense(32, activation="relu"), prob=0.01)(capsul) # 5. attention later atten = Attention(step_dim=MAX_LEN, name='attention')(x) atten = DropConnect(Dense(16, activation="relu"), prob=0.05)(atten) x = Concatenate(axis=-1)([capsul, atten]) # 6. output (sigmoid) output_layer = Dense(units=1, activation='sigmoid', name='output')(x) model = Model(inputs=input_layer, outputs=output_layer) # compile model model.compile(loss='binary_crossentropy', optimizer='adam') return model
def discriminator(self): if self.D: return self.D # kern_init = initializers.RandomNormal(mean=0.0, stddev=0.02, seed=None) kern_init = initializers.glorot_normal() input_shape = (self.img_rows, self.img_cols, self.channel) input_img = Input(shape=input_shape, name='Input_Image') x = Conv2D(16, 5, strides=2, input_shape=input_shape, padding='same', kernel_initializer=kern_init)(input_img) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(32, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(64, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(128, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(256, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(512, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = LeakyReLU(alpha=0.2)(x) # Out: 1-dim probability x = Flatten()(x) x = Dense(1, activation='sigmoid')(x) self.D = Model(inputs=input_img, outputs=x, name='Discriminator') self.D.summary() return self.D
def __init__(self, sess, dim_data, dim_context, dim_konf, save_folder): self.opt_G = Adam(lr=1e-4, beta_1=0.5) self.opt_D = Adam(lr=1e-3, beta_1=0.5) self.opt_D = Adadelta() self.initializer = initializers.glorot_normal() self.sess = sess self.dim_data = dim_data self.dim_context = dim_context self.dim_k = dim_konf self.n_key_confs = dim_context[0] self.x_input = Input(shape=(dim_data, ), name='x', dtype='float32') self.w_input = Input(shape=dim_context, name='w', dtype='float32') self.k_input = Input(shape=dim_konf, name='k', dtype='float32') self.disc = self.createDisc() self.disc.summary() self.save_folder = save_folder
def ref_rnn(input_shape, n_class, model_size_info): ''' model size info[-1]: LSTM for 0 and GRU for 1 ''' cprint('**** ref_rnn ****', 'red') lstm1 = model_size_info[0] type_choice = model_size_info[-1] init = initializers.glorot_normal() # model model = Sequential() model.add( layers.Reshape(( input_shape[0], input_shape[1], ), input_shape=input_shape)) if type_choice == 0: model.add(layers.LSTM(lstm1)) elif type_choice == 1: model.add(layers.GRU(lstm1)) else: raise ValueError('wrong type name') if len(model_size_info) == 3: lstm1, dnn1, type_choice = model_size_info model.add( Dense(dnn1, activation='relu', kernel_initializer=init, bias_initializer='zeros')) model.add( Dense(n_class, activation='softmax', kernel_initializer=init, bias_initializer='zeros')) elif len(model_size_info) == 2: lstm1, type_choice = model_size_info model.add( Dense(n_class, activation='softmax', kernel_initializer=init, bias_initializer='zeros')) else: raise ValueError('model size length too long.') return model
def generator(self): if self.G: return self.G # kern_init = initializers.RandomNormal(mean=0.0, stddev=0.02, seed=None) kern_init = initializers.glorot_normal() input_shape = (self.noise_dim,) input_noise = Input(shape=input_shape, name='noise') dim = 7 # cifar10 & celebA depth = 512 x = Dense(dim * dim * depth, kernel_initializer=kern_init)(input_noise) # x = BatchNormalization()(x) # x = Activation('relu')(x) x = Reshape((dim, dim, depth))(x) x = Conv2DTranspose(depth / 2, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = Activation('selu')(x) x = Conv2DTranspose(depth / 4, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = Activation('selu')(x) x = Conv2DTranspose(depth / 8, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = Activation('selu')(x) x = Conv2DTranspose(depth / 16, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) # x = BatchNormalization()(x) x = Activation('selu')(x) x = Conv2DTranspose(self.channel, 5, strides=2, padding='same', kernel_initializer=kern_init)(x) x = Activation('tanh')(x) self.G = Model(inputs = input_noise, outputs = x, name='Generator') self.G.summary() return self.G
def get_model(embed_weights): input_layer = Input(shape=(MAX_LEN, ), name='input') # 1. embedding layer # get embedding weights print('load pre-trained embedding weights ......') input_dim = embed_weights.shape[0] output_dim = embed_weights.shape[1] x = Embedding(input_dim=input_dim, output_dim=output_dim, weights=[embed_weights], trainable=False, name='embedding')(input_layer) # clean up del embed_weights, input_dim, output_dim gc.collect() # 2. dropout x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x) # 3. bidirectional lstm x = Bidirectional(layer=CuDNNLSTM( RNN_UNITS, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm')(x) # 4. capsule layer x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True, name='capsule')(x) x = Flatten(name='flatten')(x) # # 5. dense with dropConnect # x = DropConnect( # Dense(DENSE_UNITS, activation="relu"), # prob=0.05, # name='dropConnect_dense')(x) # 6. output (sigmoid) output_layer = Dense(units=1, activation='sigmoid', name='output')(x) model = Model(inputs=input_layer, outputs=output_layer) # compile model model.compile(loss='binary_crossentropy', optimizer='adam') return model
def get_discriminator(input_size, G, G_input_size): """Returns the discriminator networks `D` and `DG`. `input_size` is the input size of `D` and `DG`. `G` is the generator model. Return a tuple of 2 elements: * The first element is `D`, the discriminator/critic * The second element id `DG` -> D(G(z))""" xavier = initializers.glorot_normal() x = Input(shape=(input_size,), name='input_x') a = Dense(256, kernel_initializer=xavier)(x) a = LeakyReLU()(a) a = Dropout(0.5)(a) a = Dense(256, kernel_initializer=xavier)(a) a = LeakyReLU()(a) a = Dropout(0.5)(a) a = Dense(16, kernel_initializer=xavier)(a) a = LeakyReLU()(a) a = Dropout(0.5)(a) # creates an output to determine if an input is fake is_fake = Dense(1, activation='linear', name='output_is_fake')(a) # creates a D model that receives a real example as input D = Model(inputs=[x], outputs=[is_fake], name='D') # creates another model that uses G as input z = Input(shape=(G_input_size,), name='D_input_z') is_fake = D(G(inputs=[z])) DG = Model(inputs=[z], outputs=[is_fake], name='DG') # D shouldn't be trained during the generator's training faze DG.get_layer('D').trainable = False DG.compile(optimizer=RMSprop(lr=1e-3), loss=[mean]) D.trainable = True D.compile(optimizer=RMSprop(lr=1e-3), loss=[wasserstein]) return D, DG
def ref_cnn(input_shape, n_class, model_size_info): ''' model_size_info: CNN1(channel, kernel, stride (time-freq)) + CNN2 + L + FC ''' # model size CNN1 = model_size_info[:5] CNN2 = model_size_info[5:10] L_size = model_size_info[-2] FC_size = model_size_info[-1] # start cnn_init = initializers.glorot_normal() model = Sequential() model.add( Conv2D(CNN1[0], kernel_size=(CNN1[1], CNN1[2]), strides=(CNN1[3], CNN1[4]), activation='relu', input_shape=input_shape, kernel_initializer=cnn_init, padding='valid')) #model.add(layers.BatchNormalization()) #model.add(Dropout(0.5)) model.add(MaxPooling2D(pool_size=(1, 2), strides=(1, 2))) model.add( Conv2D(CNN2[0], kernel_size=(CNN2[1], CNN2[2]), strides=(CNN2[3], CNN2[4]), activation='relu', kernel_initializer=cnn_init, padding='valid')) #model.add(layers.BatchNormalization()) #model.add(Dropout(0.5)) # batch norm dropout #model.add(Conv2D(CNNChannel, (CNNkernel,CNNkernel), activation='relu',kernel_initializer=cnn_init, padding='valid')) # batch norm dropout model.add(Flatten()) model.add(Dense(L_size, activation='relu')) model.add(Dense(FC_size, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(n_class, activation='softmax')) return model
def training_data(tmpdir_factory): import h5py from keras.layers import Dense from keras.models import Sequential from keras.optimizers import SGD from keras.initializers import glorot_normal, normal from deepreplay.datasets.parabola import load_data from deepreplay.callbacks import ReplayData filename = str(tmpdir_factory.mktemp('data').join('training.h5')) X, y = load_data(xlim=(-1, 1), n_points=1000, shuffle=True, seed=13) sgd = SGD(lr=0.05) glorot_initializer = glorot_normal(seed=42) normal_initializer = normal(seed=42) replaydata = ReplayData(X, y, filename=filename, group_name='part1_activation_functions') model = Sequential() model.add(Dense(input_dim=2, units=2, kernel_initializer=glorot_initializer, activation='sigmoid', name='hidden')) model.add(Dense(units=1, kernel_initializer=normal_initializer, activation='sigmoid', name='output')) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['acc']) model.fit(X, y, epochs=20, batch_size=16, callbacks=[replaydata]) training_data = h5py.File(filename, 'r') return training_data['part1_activation_functions']
def build_fc_model(img_shape): initializer = glorot_normal() x0 = Input(img_shape, name='Input') raw_x = x0 flattened_raw_x = Flatten()(raw_x) fc0 = Dense(num_fc_0, activation='relu', name='fully_informed_nodes', kernel_initializer=initializer, kernel_regularizer=l2(.001))(flattened_raw_x) fc1 = Dense(num_fc_1, activation='relu', name='dense_encoding', kernel_initializer=initializer, kernel_regularizer=l2(.001))(fc0) y = Dense(1, name='softmax', activation='sigmoid')(fc1) embedding_model = Model(inputs=x0, outputs=fc1) model = Model(inputs=x0, outputs=y) return model, embedding_model
def build_my_model(m, n): inp = Input(shape=(m, )) x = SpatialDropout1D(rate=0.24)(inp) x = Bidirectional(CuDNNLSTM(80, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x) x_1 = Attention(m)(x) x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1) x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) x_2 = Flatten()(x_2) x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2) conc = concatenate([x_1, x_2]) # conc = add([x_1, x_2]) outp = Dense(1, activation="sigmoid")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=[f1]) return model
def parse_nn_wout_json(self, modelfile): with CustomObjectScope({'GlorotUniform': glorot_uniform()}): with CustomObjectScope({'GlorotNormal': glorot_normal()}): try: model = models.load_model(modelfile) except: pass try: model = loadmodel(modelfile) except: print( 'We cannot load the model, make sure the keras file was saved in a supported version' ) print(err) [nl, ni, no] = self.get_shape(model) [lys, lfs] = self.get_layers(model, nl) #lfs = self.fix_activations(lys,lfs) [lsize, n, nls] = self.get_neurons(model, nl) [W, b] = self.get_parameters(model, nl, nls) return self.save_nnmat_file(model, ni, no, nls, n, lsize, W, b, lys, lfs)
def attention_capsule(maxlen, max_features, embed_size, embedding_matrix, num_classes): inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(rate=0.24)(x) x = Bidirectional(LSTM(80, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x) x_1 = Attention(maxlen)(x) x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1) x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) x_2 = Flatten()(x_2) x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2) conc = concatenate([x_1, x_2]) # conc = add([x_1, x_2]) outp = Dense(num_classes, activation="sigmoid")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy']) return model
from deepreplay.callbacks import ReplayData from deepreplay.replay import Replay from deepreplay.plot import compose_animations, compose_plots from sklearn.datasets import make_moons import matplotlib.pyplot as plt group_name = 'moons' X, y = make_moons(n_samples=2000, random_state=27, noise=0.03) sgd = SGD(lr=0.01) glorot_initializer = glorot_normal(seed=42) normal_initializer = normal(seed=42) replaydata = ReplayData(X, y, filename='moons_dataset.h5', group_name=group_name) model = Sequential() model.add(Dense(input_dim=2, units=4, kernel_initializer=glorot_initializer, activation='tanh')) model.add(Dense(units=2, kernel_initializer=glorot_initializer, activation='tanh', name='hidden')) model.add(Dense(units=1,
def test_glorot_normal(tensor_shape): fan_in, fan_out = initializers._compute_fans(tensor_shape) std = np.sqrt(2. / (fan_in + fan_out)) _runner(initializers.glorot_normal(), tensor_shape, target_mean=0., target_std=std)
def _make_model(self): if self.is_hgg: dropout_rate = 0.1 else: dropout_rate = 0.5 step = 0 print('******************************************', step) step += 1 model_to_make = Sequential() print('******************************************', step) step += 1 model_to_make.add(Conv2D(64, (3, 3), kernel_initializer=glorot_normal(), bias_initializer='zeros', padding='same', data_format='channels_first', input_shape=(4, 33, 33) )) print(model_to_make.input_shape) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add(LeakyReLU(alpha=0.333)) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add(Conv2D(filters=64, kernel_size=(3, 3), padding='same', data_format='channels_first', input_shape=(64, 33, 33))) print(model_to_make.output) print('******************************************', step) step += 1 model_to_make.add(LeakyReLU(alpha=0.333)) print(model_to_make.output) if self.is_hgg: model_to_make.add(Conv2D(filters=64, kernel_size=(3, 3), padding='same', data_format='channels_first', input_shape=(64, 33, 33))) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(LeakyReLU(alpha=0.333)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(MaxPool2D(pool_size=(3, 3), strides=(2, 2), data_format='channels_first', input_shape=(64, 33, 33))) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same', data_format='channels_first', input_shape=(64, 16, 16))) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(LeakyReLU(alpha=0.333)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same', data_format='channels_first', input_shape=(128, 16, 16))) print('******************************************', step) step += 1 print(model_to_make.output) if self.is_hgg: model_to_make.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same', data_format='channels_first', input_shape=(128, 16, 16))) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(LeakyReLU(alpha=0.333)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(MaxPool2D(pool_size=(3, 3), strides=(2, 2), data_format='channels_first', input_shape=(128, 16, 16))) print('******************************************', step) step += 1 print(model_to_make.output) print('******************************************', 'flattened') model_to_make.add(Flatten()) print(model_to_make.output) model_to_make.add(Dense(units=256, input_dim=6272)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(LeakyReLU(alpha=0.333)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Dropout(dropout_rate)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Dense(units=256, input_dim=256)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(LeakyReLU(alpha=0.333)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Dropout(dropout_rate)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Dense(units=5, input_dim=256)) print('******************************************', step) step += 1 print(model_to_make.output) model_to_make.add(Activation('softmax')) print('******************************************', step) step += 1 print(model_to_make.output) self.model = model_to_make