def ctpn(base_features, num_anchors, rnn_units=128, fc_units=512): """ ctpn网络 :param base_features: (B,H,W,C) :param num_anchors: anchors个数 :param rnn_units: :param fc_units: :return: """ x = layers.Conv2D(512, kernel_size=(3, 3), padding='same', name='pre_fc')(base_features) # [B,H,W,512] # 沿着宽度方式做rnn rnn_forward = layers.TimeDistributed(layers.GRU(rnn_units, return_sequences=True, kernel_initializer='he_normal'), name='gru_forward')(x) rnn_backward = layers.TimeDistributed( layers.GRU(rnn_units, return_sequences=True, kernel_initializer='he_normal', go_backwards=True), name='gru_backward')(x) rnn_output = layers.Concatenate(name='gru_concat')([rnn_forward, rnn_backward]) # (B,H,W,256) # conv实现fc fc_output = layers.Conv2D(fc_units, kernel_size=(1, 1), activation='relu', name='fc_output')( rnn_output) # (B,H,W,512) # 分类 class_logits = layers.Conv2D(2 * num_anchors, kernel_size=(1, 1), name='cls')(fc_output) class_logits = layers.Reshape(target_shape=(-1, 2), name='cls_reshape')(class_logits) # 中心点垂直坐标和高度回归 predict_deltas = layers.Conv2D(2 * num_anchors, kernel_size=(1, 1), name='deltas')(fc_output) predict_deltas = layers.Reshape(target_shape=(-1, 2), name='deltas_reshape')(predict_deltas) # 侧边精调(只需要预测x偏移即可) predict_side_deltas = layers.Conv2D(num_anchors, kernel_size=(1, 1), name='side_deltas')(fc_output) predict_side_deltas = layers.Reshape(target_shape=(-1, 1), name='side_deltas_reshape')( predict_side_deltas) return class_logits, predict_deltas, predict_side_deltas
def cnn_2x_encdec_siamese(voc_size, max_len, dropout=0.5): """Two siamese branches, each embedding a statement. Binary classifier on top. Args: voc_size: size of the vocabulary for the input statements. max_len: maximum length for the input statements. Returns: A Keras model instance. """ pivot_input = layers.Input(shape=(max_len, ), dtype='int32') statement_input = layers.Input(shape=(max_len, ), dtype='int32') x = layers.Embedding(output_dim=256, input_dim=voc_size, input_length=max_len)(pivot_input) x = layers.Convolution1D(256, 7, activation='relu')(x) x = layers.MaxPooling1D(3)(x) x = layers.Convolution1D(256, 7, activation='relu')(x) x = layers.MaxPooling1D(5)(x) x = layers.GRU(256)(x) #this acts as the encoder x = layers.RepeatVector(256)( x) #take the last output of GRU and feed it to the decoder embedded_pivot = layers.GRU(256)(x) encoder_model = Model(pivot_input, embedded_pivot) embedded_statement = encoder_model(statement_input) concat = layers.merge([embedded_pivot, embedded_statement], mode='concat') x = layers.Dense(256, activation='tanh')(concat) x = layers.Dropout(dropout)(x) prediction = layers.Dense(1, activation='sigmoid')(x) model = Model([pivot_input, statement_input], prediction) return model
def main(): float_data = get_data('jena_climate_2009_2016.csv') lookback = 1440 step = 6 delay = 144 batch_size = 128 data = normalize(float_data[:200000]) train_gen = generator(data, lookback=lookback, delay=delay, min_index=None, max_index=200000, step=step, batch_size=batch_size) val_gen = generator(data, lookback=lookback, delay=delay, min_index=200001, max_index=300000, step=step, batch_size=batch_size) test_gen = generator(data, lookback=lookback, delay=delay, min_index=300001, max_index=None, step=step, batch_size=batch_size) model = Sequential() model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5, return_sequences=True, input_shape=(None, data.shape[-1]))) model.add(layers.GRU(64, activation='relu', dropout=0.1, recurrent_dropout=0.5)) model.add(layers.Dense(1)) model.compile(optimizer=RMSprop(), loss='mae') val_steps = (300000 - 200001 - lookback) history = model.fit_generator(train_gen, steps_per_epoch=500, epochs=40, validation_data=val_gen, validation_steps=val_steps, callbacks=model_callbacks()) epochs = range(1, len(loss) + 1) loss = model_history.history['loss'] val_loss = model_history.history['val_loss'] plt.figure() plt.plot(epochs, loss, 'bo', label='Training loss') plt.plot(epochs, val_loss, 'b', label='Validation loss') plt.show()
def build_decoder(self): """ Build a decoder that transforms latent representations into decoded outputs. """ self.latent_inputs = Input(shape=(self.latent_dim, ), name='Vhod_v_dekodirnik') x = layers.Dense(self.latent_dim, activation='tanh', name="Polno_povezan_sloj_2")(self.latent_inputs) x = layers.Dropout(self.dropout_rate_mid, name="Izpustveni_sloj_2")(x) x = layers.BatchNormalization(axis=-1, name="Paketna_normalizacija_5")(x) x = layers.RepeatVector(self.compound_length, name="Ponovi_vektor")(x) x = layers.GRU(self.gru_size, activation='tanh', return_sequences=True, name="GRU_sloj_1")(x) x = layers.GRU(self.gru_size, activation='tanh', return_sequences=True, name="GRU_sloj_2")(x) x = layers.GRU(self.gru_size, activation='tanh', return_sequences=True, name="GRU_sloj_3")(x) x = layers.GRU(self.charset_length, return_sequences=True, activation='softmax')(x) self.outputs = x
def transfer_deeprnn(models: List[Model] = None) -> Model: # acc(train/valid/test): 0.87/0.88/0.862 | 2 epochs, commit ad1e | Adam lr 0.001 # Pre: models[0] is birnn birnn = BaseModels.birnn() if models is None else models[0] for i in range(4): birnn.layers.pop() X = birnn.layers[-1].output for layer in birnn.layers: layer.trainable = False X = layers.Dropout(.5)(X) X = layers.Bidirectional( layers.GRU(units=128, dropout=.2, recurrent_dropout=.2, return_sequences=True))(X) X = layers.Bidirectional( layers.GRU(units=128, dropout=.2, recurrent_dropout=.2))(X) X = layers.Dropout(.5)(X) X = layers.Dense(64, activation='relu')(X) X = layers.Dense(1, activation='sigmoid')(X) return Model(inputs=birnn.inputs, outputs=X, name=_name_model('transfer-deeprnn'))
def build_RNN(input_shape, dropout=0.5, recurrent_dropout=0.5, stacked=False, bidirectional=False): try: assert isinstance(input_shape, tuple) assert isinstance(dropout, (int, float)) and 0 <= dropout < 1 assert isinstance(recurrent_dropout, (int, float)) and 0 <= recurrent_dropout < 1 assert isinstance(stacked, (bool, int)) assert isinstance(bidirectional, (bool, int)) except AssertionError: raise model = models.Sequential() if bidirectional: model.add(layers.Bidirectional( layers.GRU(32, input_shape=(None, input_shape[-1])))) model.add(layers.Dense(1)) else: model.add(layers.GRU(32, dropout=dropout, recurrent_dropout=recurrent_dropout, input_shape=(None, input_shape[-1]), return_sequences=True if stacked else False)) if stacked: model.add(layers.GRU(64, activation='relu', dropout=dropout, recurrent_dropout=recurrent_dropout)) model.add(layers.Dense(1)) model.compile(optimizer=RMSprop(lr=0.001), loss='mae') return model
def build_text_rcnn(max_words=20000, maxlen=200, embedding_dim=400, classification_type=2): sentence_input = layers.Input(shape=(maxlen,), dtype='int32') embedded_sequences = layers.Embedding(max_words, embedding_dim)(sentence_input) x_backwords = layers.GRU(100, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=True)(embedded_sequences) x_backwords_reverse = layers.Lambda(lambda x: K.reverse(x, axes=1))(x_backwords) x_fordwords = layers.GRU(100, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=False)(embedded_sequences) x_feb = layers.Concatenate(axis=2)([x_fordwords, embedded_sequences, x_backwords_reverse]) x_feb = layers.Dropout(0.32)(x_feb) # Concatenate后的embedding_size dim_2 = K.int_shape(x_feb)[2] x_feb_reshape = layers.Reshape((maxlen, dim_2, 1))(x_feb) filters = [2, 3, 4, 5] conv_pools = [] for filter in filters: conv =layers. Conv2D(filters=300, kernel_size=(filter, dim_2), padding='valid', kernel_initializer='normal', activation='relu', )(x_feb_reshape) pooled = layers.MaxPooling2D(pool_size=(maxlen - filter + 1, 1), strides=(1, 1), padding='valid', )(conv) conv_pools.append(pooled) x = layers.Concatenate()(conv_pools) x = layers.Flatten()(x) output = layers.Dense(classification_type, activation='softmax')(x) model = models.Model(sentence_input, output) return model
def model_GRU(units_1st_layer, input_shape, num_outputs, dropout_1st_layer, recurrent_dropout_1st_layer, metrics, units_2nd_layer=None, droput_2nd_layer=0.0, recurrent_dropout_2nd_layer=0.0): model = Sequential() if units_2nd_layer: model.add( layers.GRU(units_1st_layer, dropout=dropout_1st_layer, recurrent_dropout=recurrent_dropout_1st_layer, return_sequences=True, input_shape=input_shape)) model.add( layers.GRU(units_2nd_layer, activation='relu', dropout=droput_2nd_layer, recurrent_dropout=recurrent_dropout_2nd_layer)) else: model.add( layers.GRU(units_1st_layer, dropout=dropout_1st_layer, recurrent_dropout=recurrent_dropout_1st_layer, input_shape=input_shape)) model.add(layers.Dense(num_outputs, activation='sigmoid')) model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=metrics) return model
def model_GRU(): model = Sequential() model.add( layers.GRU(64, dropout=0.1, recurrent_dropout=0.6, return_sequences=True, input_shape=(600, gene_data.shape[-1]))) # model.add(layers.Dense(64, activation='relu')) model.add( layers.GRU(32, dropout=0.1, recurrent_dropout=0.5, return_sequences=True, input_shape=(600, gene_data.shape[-1]))) # model.add(layers.Dense(32, activation='relu')) model.add( layers.GRU(16, dropout=0.1, recurrent_dropout=0.4, return_sequences=True, input_shape=(32, gene_data.shape[-1]))) model.add(layers.Flatten()) model.add(layers.Dense(8, activation='relu')) # model.add(layers.Flatten()) # model.add(layers.Dense(32, activation='relu')) # model.add(layers.Dense(8, activation='relu')) model.add(layers.Dense(3, activation='softmax')) # model.add(Activation('sigmoid')) return model
def gru_meatod_dropout_2(float_data, train_gen, val_gen, val_steps): model = models.Sequential() model.add( layers.GRU( 32, input_shape=(None, float_data.shape[-1]), dropout=0.1, recurrent_dropout= 0.3, # dropout,指定该层输入单元的dropout比率,recurrent_dropout指定循环单元的dropout比率 return_sequences=True)) model.add( layers.GRU( 32, input_shape=(None, float_data.shape[-1]), dropout=0.1, recurrent_dropout= 0.3, # dropout,指定该层输入单元的dropout比率,recurrent_dropout指定循环单元的dropout比率 return_sequences=True)) model.add( layers.GRU(64, dropout=0.1, recurrent_dropout=0.3, activation='relu')) model.add(layers.Dense(32)) model.add(layers.Dense(1)) model.compile(loss='mae', optimizer=optimizers.RMSprop()) history = model.fit_generator( train_gen, # 数据生产器 steps_per_epoch=500, # 每轮抽取多少批次的生成器的数据,每批次128,共200000, epochs=20, # 训练轮次 validation_data=val_gen, # 验证集,可以是numpy数组组成的元祖,也可以是数据生成器 validation_steps=val_steps # 从验证集中抽取多少个批次用于评估 ) return history
def build_neural_net(hyperparameters): # the size of the embedding vector embedding_vector_size: int = hyperparameters[2] # whether to use a bidirectional RNN over the embedding use_bidirectional: bool = hyperparameters[3] # the vector size representing the input sequence input_vector_representation_size: int = hyperparameters[4] # whether to use a SimpleRNN, LSTM or GRU use_SRNN_LSTM_GRU: str = hyperparameters[5] # whether to use a second RNN after the first use_second_RNN: bool = hyperparameters[6] # the vector size of the second RNN intermediate_vector_representation_size: int = hyperparameters[7] # rate of dropout to use - float between 0 and 1 dropout: float = hyperparameters[8] model = models.Sequential() # Add an Embedding layer to the model, with as many inputs as terms in the vocab, # and as many nodes as defined by the embedding_vector_size hyper parameter model.add(layers.Embedding(len(vocab), embedding_vector_size, input_length=None, mask_zero=True)) # Add the first RNN Layer. If the use_bidirectional hyper parameter is set to True, # then use a bidirectional implementation if use_bidirectional: # Add the first RNN Layer as a Simple RNN, LSTM or GRU depending on the use_SRNN_LSTM_GRU hyper parameter # also use dropoput according to the hyper parameter # and return sequences of the first layer depending on whether a second Recursive Layer will be used if use_SRNN_LSTM_GRU == 'SRNN': model.add(layers.Bidirectional(layers.SimpleRNN(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN))) elif use_SRNN_LSTM_GRU == 'LSTM': model.add(layers.Bidirectional(layers.LSTM(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN))) elif use_SRNN_LSTM_GRU == 'GRU': model.add(layers.Bidirectional(layers.GRU(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN))) else: if use_SRNN_LSTM_GRU == 'SRNN': model.add(layers.SimpleRNN(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN)) elif use_SRNN_LSTM_GRU == 'LSTM': model.add(layers.LSTM(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN)) elif use_SRNN_LSTM_GRU == 'GRU': model.add(layers.GRU(input_vector_representation_size, dropout=dropout, return_sequences=use_second_RNN)) if use_second_RNN: if use_SRNN_LSTM_GRU == 'SRNN': model.add(layers.SimpleRNN(intermediate_vector_representation_size, dropout=dropout)) elif use_SRNN_LSTM_GRU == 'LSTM': model.add(layers.LSTM(intermediate_vector_representation_size, dropout=dropout)) elif use_SRNN_LSTM_GRU == 'GRU': model.add(layers.GRU(intermediate_vector_representation_size, dropout=dropout)) # softmax layer model.add(layers.Dense(19, activation='softmax')) return model
def model_3(self): embedding_matrix = self.build_myself_embedding_matrix() input = layers.Input(shape=(self.max_words,)) embedding = layers.Embedding(input_dim=embedding_matrix.shape[0], output_dim=embedding_matrix.shape[1], input_length=self.max_words, weights=[embedding_matrix]) x = layers.SpatialDropout1D(0.2)(embedding(input)) x = layers.Bidirectional(layers.GRU(400, return_sequences=True))(x) x = layers.Bidirectional(layers.GRU(400, return_sequences=True))(x) avg_pool = layers.GlobalAveragePooling1D()(x) max_pool = layers.GlobalMaxPool1D()(x) concat = layers.concatenate([avg_pool, max_pool]) x = layers.Dense(1024)(concat) x = layers.BatchNormalization()(x) x = layers.Activation(activation='relu')(x) x = layers.Dropout(0.2)(x) x = layers.Dense(512)(x) x = layers.BatchNormalization()(x) x = layers.Activation(activation='relu')(x) x = layers.Dropout(0.2)(x) output = layers.Dense(self.class_num, activation='softmax')(x) model = models.Model(input=input, output=output) print(model.summary()) return model
def build_model(shape, vocab_size): """构建模型 """ input_layer = layers.Input(shape=shape) m = input_layer m = layers.Embedding(vocab_size, 64)(m) m = layers.Dropout(0.1)(m) m = layers.GRU( 32, return_sequences=True, # recurrent_dropout=0.2, kernel_regularizer=regularizers.l2(0.001))(m) m = layers.GRU( 32, return_sequences=True, # recurrent_dropout=0.2, kernel_regularizer=regularizers.l2(0.001))(m) atten = m atten = layers.Flatten()(atten) atten = layers.Dense(shape[0], activation='softmax')(atten) atten = layers.RepeatVector(32)(atten) atten = layers.Permute((2, 1))(atten) m = layers.Multiply()([m, atten]) # m = layers.Add()([m, emb]) m = layers.Flatten()(m) m = layers.GaussianNoise(0.01)(m) m = layers.Dense(300, activation='linear', kernel_regularizer=regularizers.l2(0.01))(m) m = layers.BatchNormalization()(m) m = layers.Activation('tanh')(m) m = layers.Dropout(0.4)(m) m = layers.Dense(300, activation='linear', kernel_regularizer=regularizers.l2(0.01))(m) m = layers.BatchNormalization()(m) m = layers.Activation('tanh')(m) m = layers.Dropout(0.4)(m) m = layers.Dense(len(LABEL_DICT), activation='softmax')(m) atten_model = models.Model(inputs=[input_layer], outputs=atten) model = models.Model(inputs=[input_layer], outputs=m) optimizer = optimizers.Adam(lr=0.001, clipnorm=5.) model.compile(optimizer, 'categorical_crossentropy', metrics=['accuracy']) model.summary() return model, atten_model
def iter_gru_model(): """使用Dropout正则化的GRU堆叠层""" model = models.Sequential() model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5, return_sequences=True, input_shape=(None, float_data.shape[-1]))) model.add(layers.GRU(32, dropout=0.2, recurrent_dropout=0.5)) model.add(layers.Dense(1)) return model
def build_model(GRU_SIZE=1024, WORDVEC_SIZE=300, ACTIVATION='relu'): resnet = build_resnet() # Global Image featuers (convnet output for the whole image) input_img_global = layers.Input(shape=IMG_SHAPE) image_global = resnet(input_img_global) image_global = layers.BatchNormalization()(image_global) image_global = layers.Dense(WORDVEC_SIZE / 2, activation=ACTIVATION)(image_global) image_global = layers.BatchNormalization()(image_global) image_global = layers.RepeatVector(MAX_WORDS)(image_global) # Local Image features (convnet output inside the bounding box) input_img_local = layers.Input(shape=IMG_SHAPE) image_local = resnet(input_img_local) image_local = layers.BatchNormalization()(image_local) image_local = layers.Dense(WORDVEC_SIZE / 2, activation=ACTIVATION)(image_local) image_local = layers.BatchNormalization()(image_local) image_local = layers.RepeatVector(MAX_WORDS)(image_local) # Context Vector input # normalized to [0,1] the values: # left, top, right, bottom, (box area / image area) input_ctx = layers.Input(shape=(5, )) ctx = layers.BatchNormalization()(input_ctx) ctx = layers.RepeatVector(MAX_WORDS)(ctx) language_model = models.Sequential() input_words = layers.Input(shape=(MAX_WORDS, ), dtype='int32') language = layers.Embedding(words.VOCABULARY_SIZE, WORDVEC_SIZE, input_length=MAX_WORDS)(input_words) language = layers.BatchNormalization()(language) language = layers.GRU(GRU_SIZE, return_sequences=True)(language) language = layers.BatchNormalization()(language) language = layers.TimeDistributed( layers.Dense(WORDVEC_SIZE, activation=ACTIVATION))(language) language = layers.BatchNormalization()(language) # Problem with Keras 2: # TypeError: Tensors in list passed to 'values' of 'ConcatV2' Op have types [uint8, uint8, bool, uint8] that don't all match. # Masking doesn't work along with concatenation. # How do I get mask_zero=True working in the embed layer? x = layers.concatenate([image_global, image_local, ctx, language]) x = layers.GRU(GRU_SIZE)(x) x = layers.BatchNormalization()(x) x = layers.Dense(words.VOCABULARY_SIZE, activation='softmax')(x) return models.Model( inputs=[input_img_global, input_img_local, input_words, input_ctx], outputs=x)
def create_model(x, y, maxlen, epochs): """ Creates and trains a model. :param x: Numpy boolean array of shape (Number of sequences, maxlen, number of distinct character) :type x: numpy.ndarray :param y: Numpy boolean array of shape (Number of sequences, number of distinct character) :type y: numpy.ndarray :param maxlen: the length of a sequence to extract as train :type maxlen: int :param epochs: number of training iterations :type epochs: int :param chars: list of unique characters :type chars: list :returns: trained keras model :rtype: keras.engine.sequential.Sequential """ # model = Sequential() # model.add(layers.GRU( # 32, # return_sequences=True, # input_shape=(maxlen, len(chars))) # ) # model.add(layers.GRU( # 64, # input_shape=(maxlen, len(chars))) # ) # model.add(layers.Dense( # len(chars), # activation='softmax') # ) # start of my model attempt, it works decently well # - try commenting it out and using the previous one # - also try removing the Dropout layers # -------------------------------------------- model = Sequential() model.add( layers.GRU(256, return_sequences=True, input_shape=(maxlen, num_vals))) model.add(layers.Dropout(0.5)) model.add(layers.GRU(128)) model.add(layers.Dropout(0.5)) model.add(layers.Dense( num_vals, activation='softmax')) #alternatively: activation = 'softmax' model.compile(loss='categorical_crossentropy', optimizer='rmsprop') # -------------------------------------------- print(model.summary()) model.fit(x, y, batch_size=128, epochs=epochs) return (model)
def create_gru_model(): model = Sequential() model.add(layers.GRU(64, dropout=0.1, recurrent_dropout=0.4, return_sequences=True, input_shape=(None, scal.FEATURE_NUM))) model.add(layers.GRU(128,activation='relu',dropout=0.1,recurrent_dropout=0.4)) model.add(layers.Dense(1, activation='sigmoid')) model.compile(optimizer=RMSprop(),loss='binary_crossentropy') return model
def _build_gru(self): inp = KL.Input(shape=(100, self.feature_size)) x = inp x = KL.Masking()(x) x = KL.GRU(32, return_sequences=True)(x) x = KL.GRU(1)(x) out = x model = keras.Model(inp, out) model.compile(optimizer=keras.optimizers.Adam(lr=0.001), loss='mse') return model
def model_jena_p218(): model = models.Sequential() model.add( layers.GRU(32, dropout=0.2, recurrent_dropout=0.2, return_sequences=True, input_shape=(None, 7))) model.add(layers.GRU(64, dropout=0.2, recurrent_dropout=0.2)) model.add(layers.Dense(1)) return model
def build_model(self,x_train,y_train,batch_size,epochs=20): timesteps, dim = x_train.shape[1],x_train.shape[2] self.model.add(layers.GRU(128,dropout=0.2,input_shape=(timesteps,dim),return_sequences=True)) self.model.add(layers.BatchNormalization()) self.model.add(layers.GRU(128,dropout=0.0,return_sequences=True)) self.model.add(layers.BatchNormalization()) self.model.add((layers.Dense(1))) print(self.model.summary()) self.model.compile(optimizer='adam',loss='mae') self.history=self.model.fit(x_train,y_train,batch_size=batch_size,validation_split=0.2,verbose=1,epochs=epochs) self.history_dict=self.history.history return self.model,self.history_dict
def train_point(self, number): model = keras.Sequential() model.add( layers.GRU(units=100, use_bias=False, input_shape=(number+1, 4), return_sequences=True, name='GRU_1') ) model.add(layers.Dropout(0.3)) model.add( layers.GRU(100, use_bias=True, return_sequences=False, name='GRU_2') ) model.add(layers.Dropout(0.3)) model.add(layers.Dense(units=1, use_bias=True, name='DENSE_1')) model.summary() model.compile(optimizer='adam', loss='mse', metrics=['accuracy']) dataManager = DataManager('D:/DataSet/stock/preparation/') train_x, train_y = dataManager.creatData_Method_2(number) index = numpy.arange(len(train_x)) numpy.random.shuffle(index) train_x = train_x[index] train_y = train_y[index] print('train epoch: ', len(train_x)) Log_dir = 'D:/Git/tensorflow_cpu/StockPractice/' file_name = 'stock_GRU_GRU_DENSE' tb = TensorBoard(log_dir=Log_dir + file_name, write_grads=True) model.fit(train_x, train_y, batch_size=256, shuffle=True, epochs=5, verbose=1) # callbacks=[tb], #validation_data=[test_x,test_y]) model.save(filepath=Log_dir+'stock_GRU_GRU_DENSE_point.h5',overwrite=True) testManager = DataManager('D:/DataSet/stock/test/') test_x,test_y = testManager.creatData_Method_2(number) print(' test epoch: ', len(test_x)) predict_y = model.predict_on_batch(test_x) plt.plot(range(len(test_y)),test_y,'-') plt.plot(range(len(test_y)), predict_y, '-') plt.show()
def transfer_learn_ocr_model(img_shape, conv_filters, kernel_size, pool_size, time_dense_size, dropout, rnn_size, act_conv,act_dense, n_classes, max_string_len, training=True): # set up VGG19 covolutional layers input_data = layers.Input(name='the_input', shape=img_shape) vgg_model = VGG16(include_top=False, weights='imagenet', input_tensor=input_data) for i in range(len(vgg_model.layers)): vgg_model.layers[i].trainable = False # get only 3 block of convolutional layers vgg_last_layer = vgg_model.layers[9].output # do recurent part # do recurent part conv_to_rnn_dims = (int(vgg_last_layer.shape[1]), int(vgg_last_layer.shape[2] * vgg_last_layer.shape[3])) inner = layers.Reshape(target_shape=conv_to_rnn_dims, name='reshape')(vgg_last_layer) # dropout1 = layers.Dropout(dropout, name='dropout1')(inner) # cuts down input size going into RNN: inner_dense = layers.Dense(time_dense_size, activation=act_dense, name='time_dense')(inner) inner_dense = layers.BatchNormalization()(inner_dense) # Two layers of bidirecitonal GRUs # GRU seems to work as well, if not better than GRU: gru_1 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1_a')(inner_dense) gru_1b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(inner_dense) gru1_merged = layers.merge([gru_1, gru_1b], mode='sum') gru1_merged = layers.BatchNormalization()(gru1_merged) gru_2 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2_a')(gru1_merged) gru_2b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged) gru_merged = layers.merge([gru_2, gru_2b], mode='concat') gru_merged = layers.BatchNormalization()(gru_merged) # dropout2 = layers.Dropout(dropout, name='dropout2')(gru_merged) # transforms RNN output to character activations: 0-9 + '/' + ' ' + null dense = layers.Dense(n_classes, kernel_initializer='he_normal', name='class_dense')(gru_merged) y_pred = layers.Activation('softmax', name='softmax')(dense) # clipnorm seems to speeds up convergence sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5) if training: # make layers for output labels = layers.Input(name='the_labels', shape=[max_string_len], dtype='float64') input_length = layers.Input(name='input_length', shape=[1], dtype='int64') label_length = layers.Input(name='label_length', shape=[1], dtype='int64') # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')( [y_pred, labels, input_length, label_length]) model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out) model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd) else: model = Model(input_data, y_pred) model.summary() return model
def train_model(gen_train, gen_valid, idx): model = models.Sequential() model.add(layers.InputLayer(input_shape=(None, 4))) model.add(layers.Masking(mask_value=0., input_shape=(None, 4))) model.add(layers.BatchNormalization()) model.add( layers.Bidirectional(layers.GRU(16, return_sequences=True), merge_mode='ave')) model.add(layers.BatchNormalization()) model.add( layers.Bidirectional(layers.GRU(16, return_sequences=True), merge_mode='ave')) model.add(layers.BatchNormalization()) model.add(layers.Dense(16)) model.add(layers.BatchNormalization()) model.add(layers.Activation('relu')) model.add(layers.Dense(2, activation='sigmoid')) model.summary() callbacks_list = [ callbacks.EarlyStopping(monitor="val_symmetric_accuracy", patience=1, min_delta=0.001, mode='max'), callbacks.ModelCheckpoint(filepath="track_%i_weights.h5" % idx, monitor="val_symmetric_accuracy", save_best_only=True, save_weights_only=True), callbacks.ReduceLROnPlateau(monitor="val_symmetric_accuracy", factor=0.5, mode='max', min_delta=0.001, patience=1) ] model.compile(optimizer=optimizers.Adam(lr=0.002), loss=losses.binary_crossentropy, metrics=[symmetric_accuracy, mask_accuracy]) out = model.fit_generator(gen_train, steps_per_epoch=len(gen_train), epochs=50, callbacks=callbacks_list, validation_data=gen_valid, validation_steps=len(gen_valid)) model.save('model_tracker_%i.h5' % idx) with open("history_track_%i.pkl" % idx, "wb") as file: pickle.dump(out.history, file)
def GRU_vol(input_dim, time_step, batch_size, epochs, activation, bias_initializer, kernel_initializer, bias_regularizer, hidden_layers, dropout, dropout_rate, batch_normalization, early_stop, x_train, y_train, lr): model = Sequential() model.add( layers.GRU(hidden_layers[0], input_shape=[time_step, input_dim], activation=activation, bias_initializer=bias_initializer, kernel_initializer=kernel_initializer, bias_regularizer=bias_regularizer, return_sequences=True)) if batch_normalization: model.add(BatchNormalization()) if dropout: model.add(Dropout(dropout_rate)) for hidden_layer in hidden_layers[1:]: model.add( layers.GRU(hidden_layer, activation=activation, bias_initializer=bias_initializer, kernel_initializer=kernel_initializer, return_sequences=True)) if batch_normalization: model.add(BatchNormalization()) if dropout: model.add(Dropout(dropout_rate)) model.add(Dense(1)) model.compile(loss=keras.losses.mse, optimizer=keras.optimizers.Adam(lr=lr)) if early_stop: model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0, callbacks=[EarlyStopping(patience=10)], validation_split=0.2) else: model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0) return model
def unit(x): ident = x x = layers.Bidirectional( GRU(Var.hidden_units, activation=Var.activationF, return_sequences=True, kernel_regularizer=regularizers.l2(Var.Kr), activity_regularizer=regularizers.l2(Var.Ar), kernel_constraint=max_norm(max_value=3.), dropout=Var.dropout, recurrent_dropout=Var.dropout))(x) x = layers.GRU(Var.hidden_units * 2, return_sequences=True, kernel_regularizer=regularizers.l2(Var.Kr), activity_regularizer=regularizers.l2(Var.Ar), kernel_constraint=max_norm(max_value=3.), dropout=Var.dropout, recurrent_dropout=Var.dropout)(x) x = layers.Bidirectional( GRU(Var.hidden_units * 2, return_sequences=True, kernel_regularizer=regularizers.l2(Var.Kr), activity_regularizer=regularizers.l2(Var.Ar), kernel_constraint=max_norm(max_value=3.), dropout=Var.dropout, recurrent_dropout=Var.dropout))(x) x = layers.GRU(Var.hidden_units * 4, return_sequences=True, kernel_regularizer=regularizers.l2(Var.Kr), activity_regularizer=regularizers.l2(Var.Ar), kernel_constraint=max_norm(max_value=3.), dropout=Var.dropout, recurrent_dropout=Var.dropout)(x) x = layers.Bidirectional( GRU(Var.hidden_units * 4, return_sequences=True, kernel_regularizer=regularizers.l2(Var.Kr), activity_regularizer=regularizers.l2(Var.Ar), kernel_constraint=max_norm(max_value=3.), dropout=Var.dropout, recurrent_dropout=Var.dropout))(x) x = layers.Dropout(Var.dropout, noise_shape=(None, 1, Var.hidden_units * 8))(x) x = layers.Dense(Var.Dense_Unit, activation=Var.activationF, kernel_constraint=max_norm(max_value=3.))(x) # x = merge([ident,x], mode = 'sum') #mode 'sum' concat x = layers.add([ident, x]) return x
def buildGRUModel(input_size, seq_len, hidden_size): model = km.Sequential() model.add( kl.GRU(input_dim=input_size, output_dim=hidden_size, return_sequences=False)) model.add(kl.Dense(hidden_size, activation="relu")) model.add(kl.RepeatVector(seq_len)) model.add(kl.GRU(hidden_size, return_sequences=True)) model.add( kl.TimeDistributed(kl.Dense(output_dim=input_size, activation="linear"))) model.compile(loss="mse", optimizer='adam') return model
def create_model(): # Define a model that utilizes bidirectional GRU/LSTM celss model = keras.models.Sequential() model.add(L.InputLayer([None], dtype='int32')) model.add(L.Embedding(len(all_words), 64)) model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True))) model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True))) model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True))) # add top layer that predicts tag probabilities model.add(L.TimeDistributed(L.Dense(len(all_tags), activation='softmax'))) return model
def train(): stims, resps = ld.load_data(['Noise 1','Long Square','Short Square']) time_steps = stims.shape[1] repeats = stims.shape[0] model = km.Sequential([ kl.GRU(50, return_sequences=True, input_shape=(time_steps,1)), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.GRU(50, return_sequences=True), kl.Dense(1) ]) model.compile(loss='mse', optimizer='adam', metrics=['accuracy']) stims = stims.reshape(stims.shape[0], stims.shape[1], 1) resps = resps.reshape(resps.shape[0], resps.shape[1], 1) model.fit(stims, resps, epochs=1000, batch_size=10, callbacks=[ kc.ModelCheckpoint('output/model_lstm.h5') ])
def vanilla_ocr_model(img_shape, conv_filters, kernel_size, pool_size, time_dense_size, dropout, rnn_size, act_conv, act_dense, n_classes, max_string_len): input_data = layers.Input(name='the_input', shape=img_shape) inner = layers.Conv2D(conv_filters, kernel_size, padding='same', activation=act_conv, name='conv1_1')(input_data) inner = layers.MaxPooling2D(pool_size=pool_size, name='max1')(inner) inner = layers.Conv2D(conv_filters, kernel_size, padding='same', activation=act_conv, name='conv2_1')(inner) inner = layers.MaxPooling2D(pool_size=pool_size, name='max2')(inner) # reshape along conv and width conv_to_rnn_dims = (int(inner.shape[1]), int(inner.shape[2] * inner.shape[3])) inner = layers.Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner) # dropout1 = layers.Dropout(dropout, name='dropout1')(inner) # cuts down input size going into RNN: inner_dense = layers.Dense(time_dense_size, activation=act_dense, name='time_dense')(inner) inner_dense = layers.BatchNormalization()(inner_dense) # Two layers of bidirecitonal GRUs # GRU seems to work as well, if not better than GRU: gru_1 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1_a')(inner_dense) gru_1b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(inner_dense) gru1_merged = layers.merge([gru_1, gru_1b], mode='sum') gru1_merged = layers.BatchNormalization()(gru1_merged) gru_2 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2_a')(gru1_merged) gru_2b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged) gru_merged = layers.merge([gru_2, gru_2b], mode='concat') gru_merged = layers.BatchNormalization()(gru_merged) # dropout2 = layers.Dropout(dropout, name='dropout2')(gru_merged) # transforms RNN output to character activations: 0-9 + '/' + ' ' + null dense = layers.Dense(n_classes, kernel_initializer='he_normal', name='class_dense')(gru_merged) y_pred = layers.Activation('softmax', name='softmax')(dense) # show model Model(inputs=input_data, outputs=y_pred).summary() # make layers for output labels = layers.Input(name='the_labels', shape=[max_string_len], dtype='float64') input_length = layers.Input(name='input_length', shape=[1], dtype='int64') label_length = layers.Input(name='label_length', shape=[1], dtype='int64') # Keras doesn't currently support loss funcs with extra parameters # so CTC loss is implemented in a lambda layer loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')( [y_pred, labels, input_length, label_length]) # clipnorm seems to speeds up convergence sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5) # loss with CTC model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out) model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd) return model
def build_han(maxlen, max_sent_len, max_words, embedding_dim, classification_type): embedding_layer = layers.Embedding(max_words, embedding_dim, input_length=maxlen) sentence_input = layers.Input(shape=(maxlen,), dtype='int32') embedded_sequences = embedding_layer(sentence_input) l_lstm = layers.Bidirectional(layers.GRU(100, return_sequences=True))(embedded_sequences) l_att = Word_Attention(100)(l_lstm) sentEncoder = models.Model(sentence_input, l_att) sentEncoder.summary() review_input = layers.Input(shape=(max_sent_len, maxlen), dtype='int32') review_encoder = layers.TimeDistributed(sentEncoder)(review_input) l_lstm_sent = layers.Bidirectional(layers.GRU(100, return_sequences=True))(review_encoder) l_att_sent = Word_Attention(100)(l_lstm_sent) preds = layers.Dense(classification_type, activation='softmax')(l_att_sent) model = models.Model(review_input, preds) return model