from keras import Input # build network model with multi-input text_vocabulary_size = 10000 # the first 10000 words used most frequently; 10000 as the input dim in Embedding layer. question_vocabulary_size = 10000 # question text input. input one answer_vocabulary_size = 500 # answer text input. input two # input 1: text_input = Input(shape=(None, ), dtype='int32', name='text') # shape(None,): None dimension specified, The length of array input is uncertain. # name: the unique layer name in the model. embedded_text = layers.Embedding(text_vocabulary_size, 64)( text_input) # out:a word embedded into a 64 dimension vector encoded_text = layers.LSTM(32)(embedded_text) # input 2: question_input = Input(shape=(None, ), dtype='int32', name='question') # the name should be unique! embedded_question = layers.Embedding(question_vocabulary_size, 32)(question_input) encoded_question = layers.LSTM(16)(embedded_question) # connect two input tensors which could be different size. input 1 is 32 but input 2 is 16. concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1) # -1: the last axis # concatenat: https://blog.csdn.net/leviopku/article/details/82380710 answer = layers.Dense(answer_vocabulary_size, activation='softmax')( concatenated) # multi-classification
plt.legend() plt.show() ############### recurrent baseline -GRU (end) ############## ############### another recurrent baseline - LSTM ############## from keras.models import Sequential from keras import layers from keras.optimizers import RMSprop model = Sequential() model.add(layers.LSTM(4, dropout=0.5, recurrent_dropout=0.5, input_shape=(None, log_minmax_flux.shape[-1]))) model.add(layers.Dense(1)) model.summary() model.compile(optimizer=RMSprop(lr = 0.0001), loss='mae') history = model.fit_generator(train_gen, steps_per_epoch=2000, epochs=10, validation_data=val_gen, validation_steps=val_steps) import matplotlib.pyplot as plt loss = history.history['loss'] val_loss = history.history['val_loss']
encoder_inputs = layers.Input(shape=(None, )) # (1425, 79) # 임베딩 레이어 encoder_outputs = layers.Embedding( len(words), embedding_dim, input_length=max_sequences )( encoder_inputs ) # (1425, 79) -> (6406, 100, 78) // 마지막 max_sequences는 자동으로 잡아준다? 생략 가능하다. // 80 넣었을 때 에러가 날까 안날까 확인해보기 print(f'encoder_outputs : {encoder_outputs}') # return_state가 True면 상태값 리턴 # LSTM은 state_h(hidden state)와 state_c(cell state) 2개의 상태 존재 encoder_outputs, state_h, state_c = layers.LSTM( lstm_hidden_dim, # 128 노드의 개수 dropout=0.1, recurrent_dropout=0.5, return_state=True )( encoder_outputs ) # (128, 100, 78) (None, 78, 128) # 히든 상태와 셀 상태를 하나로 묶음 encoder_states = [state_h, state_c] #-------------------------------------------- # 훈련 모델 디코더 정의 #-------------------------------------------- # 목표 문장의 인덱스 시퀀스를 입력으로 받음 decoder_inputs = layers.Input(shape=(None, )) # x_decoder : (1425, 78) # 임베딩 레이어
Xtext_val, Ximage_val, ytext_val = preprocessing(dt_val, di_val) # convert input and output into array """RNN model""" from keras import layers print(vocab_size) ## image feature dim_embedding = 64 input_image = layers.Input(shape=(Ximage_train.shape[1], )) fimage = layers.Dense(256, activation='relu', name="ImageFeature")(input_image) input_txt = layers.Input(shape=(maxlen, )) ftxt = layers.Embedding(vocab_size, dim_embedding, mask_zero=True)(input_txt) ftxt = layers.LSTM(256, name="CaptionFeature")(ftxt) decoder = layers.add([ftxt, fimage]) decoder = layers.Dense(256, activation='relu')(decoder) output = layers.Dense(vocab_size, activation='softmax')(decoder) model = models.Model(inputs=[input_image, input_txt], outputs=output) model.compile(loss='categorical_crossentropy', optimizer='adam') print(model.summary()) start = time.time() hist = model.fit([Ximage_train, Xtext_train], ytext_train, epochs=5, verbose=0, batch_size=64,
from keras.datasets import imdb from keras import models from keras import layers from Utils import plotHistory, INFO from keras.preprocessing.sequence import pad_sequences max_feature = 10000 max_len = 500 print(INFO() + 'loading data') (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_feature) model = models.Sequential() model.add(layers.Embedding(max_feature, 32)) model.add(layers.LSTM(32)) model.add(layers.Dense(1, activation='sigmoid')) model.summary() print(INFO() + 'training') model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) x_train = pad_sequences(x_train,maxlen=max_len) history = model.fit(x_train, y_train, batch_size=128, epochs=10, validation_split=0.2) plotHistory(history) print(INFO() + 'end')
print('Validation Data:') print(x_val.shape) print(y_val.shape) # Try replacing GRU, or SimpleRNN. HIDDEN_SIZE = 128 BATCH_SIZE = 128 LAYERS = 1 EPOCHS = 200 print('Build model...') model = Sequential() # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE. # Note: In a situation where your input sequences have a variable length, # use input_shape=(None, num_feature). model.add(layers.LSTM(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars)))) # As the decoder RNN's input, repeatedly provide with the last hidden state of # RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum # length of output, e.g., when DIGITS=3, max output is 999+999=1998. model.add(layers.RepeatVector(MAXLEN)) # The decoder RNN could be multiple layers stacked or a single layer. for _ in range(LAYERS): # By setting return_sequences to True, return not only the last output but # all the outputs so far in the form of (num_samples, timesteps, # output_dim). This is necessary as TimeDistributed in the below expects # the first dimension to be the timesteps. model.add(layers.LSTM(HIDDEN_SIZE, return_sequences=True)) # Apply a dense layer to the every temporal slice of an input. For each of step # of the output sequence, decide which character should be chosen. model.add(layers.TimeDistributed(layers.Dense(len(chars))))
def cnn_2x_lstm_siamese(voc_size, max_len, dropout=0.2): """Two siamese branches, each embedding a statement. Binary classifier on top. Args: voc_size: size of the vocabulary for the input statements. max_len: maximum length for the input statements. dropout: Fraction of units to drop. Returns: A Keras model instance. """ pivot_input = layers.Input(shape=(max_len, ), dtype='int32') statement_input = layers.Input(shape=(max_len, ), dtype='int32') x = layers.Embedding(output_dim=256, input_dim=voc_size, input_length=max_len)(pivot_input) x = layers.Conv1D(32, 7, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(32, 7, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(64, 5, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(64, 5, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(128, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(128, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) x = layers.Conv1D(256, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.ZeroPadding1D(padding=1)(x) x = layers.Conv1D(256, 3, activation='relu', kernel_regularizer=regularizers.l2(0.01))(x) x = layers.MaxPooling1D(pool_size=2, strides=2)(x) embedded_pivot = layers.LSTM(256)(x) encoder_model = Model(pivot_input, embedded_pivot) embedded_statement = encoder_model(statement_input) concat = layers.merge([embedded_pivot, embedded_statement], mode='concat') x = layers.Dense(256, activation='relu')(concat) x = layers.Dropout(dropout)(x) prediction = layers.Dense(1, activation='relu')(x) model = Model([pivot_input, statement_input], prediction) return model
# Target features at timestep 2 [105, 106, 107, 108], # Datapoint 2 # Target features at timestep 2 [205, 206, 207, 208] ]) # Each input data point has 2 timesteps, each with 3 features. # So the input shape (excluding batch_size) is (2, 3), which # matches the shape of each data point in data_x above. model_input = L.Input(shape=(1292,128)) # This RNN will return timesteps with 4 features each. # Because return_sequences=True, it will output 2 timesteps, each # with 4 features. So the output shape (excluding batch size) is # (2, 4), which matches the shape of each data point in data_y above. model_output = L.LSTM(num_classes, return_sequences=False)(model_input) # Create the model. model = M.Model(input=model_input, output=model_output) # You need to pick appropriate loss/optimizers for your problem. # I'm just using these to make the example compile. opt = keras.optimizers.rmsprop(lr=0.0001) model.compile(optimizer=opt, loss="mean_squared_error", metrics=['accuracy']) dataPath = '/data/hibbslab/jyang/tzanetakis/ver6.0/' x_train = pickle.load(open(dataPath + 'x_train_mel.p', 'rb')) y_train = pickle.load(open(dataPath + 'y_train_mel.p', 'rb')) x_test = pickle.load(open(dataPath + 'x_test_mel.p', 'rb')) y_test = pickle.load(open(dataPath + 'y_test_mel.p', 'rb'))
sequence_input = Input(shape=(30,75)) processed_sequence = TimeDistributed(single_model)(sequence_input) pre_model = Model(inputs=sequence_input, outputs=processed_sequence) model = models.Sequential() # model.add(layers.SimpleRNN( # # unroll=True, # units=16, # batch_input_shape=(None, 50, 75),#75相当于特征数量,或者也可以叫维度,这里有3*25个坐标点,要提前整形 # return_sequences=0 # )) model.add(pre_model) model.add(layers.LSTM( units=128, batch_input_shape=(None, 30, 75), # 75相当于特征数量,或者也可以叫维度,这里有3*25个坐标点,要提前整形 return_sequences=0 # 是否返回整个列表,用于再嵌套LSTM )) # model.add(layers.Dense(16, activation='relu')) # 这个shape好像是抛去了数据的第一维度,认为这个 # model.add(layers.Dense(16, activation='relu')) # model.add(layers.Dense(8, activation='relu')) model.add(layers.Dense(40, activation='relu')) model.add(layers.Dense(40,activation='softmax')) model.summary() # plot_model(model, to_file='model.png') #编译步骤 model.compile(optimizer='rmsprop', # loss='binary_crossentropy', loss='categorical_crossentropy', metrics=['accuracy'])
def brsmv1(num_features=39, num_classes=28, num_hiddens=256, num_layers=5, dropout=0.2, zoneout=0., input_dropout=False, input_std_noise=.0, weight_decay=1e-4, residual=None, layer_norm=None, mi=None, activation='tanh'): """ BRSM v1.0 Improved features: Reference: [1] Gal, Y, "A Theoretically Grounded Application of Dropout in Recurrent Neural Networks", 2015. [2] Graves, Alex, Abdel-rahman Mohamed, and Geoffrey Hinton. "Speech recognition with deep recurrent neural networks", 2013. [3] Krueger, David, et al. "Zoneout: Regularizing rnns by randomly preserving hidden activations", 2016. [4] Ba, Jimmy Lei, Jamie Ryan Kiros, and Geoffrey E. Hinton. "Layer normalization.", 2016. [5] Wu, Yuhuai, et al. "On multiplicative integration with recurrent neural networks." Advances In Neural Information Processing Systems. 2016. [6] Wu, Yonghui, et al. "Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation.", 2016. """ x = Input(name='inputs', shape=(None, num_features)) o = x if input_std_noise is not None: o = GaussianNoise(input_std_noise)(o) if residual is not None: o = TimeDistributed( Dense(num_hiddens * 2, kernel_regularizer=l2(weight_decay)))(o) if input_dropout: o = Dropout(dropout)(o) for i, _ in enumerate(range(num_layers)): new_o = Bidirectional( keras_layers.LSTM(num_hiddens, return_sequences=True, kernel_regularizer=l2(weight_decay), recurrent_regularizer=l2(weight_decay), dropout=dropout, recurrent_dropout=dropout, activation=activation))(o) if residual is not None: o = merge([new_o, o], mode=residual) else: o = new_o o = TimeDistributed(Dense(num_classes, kernel_regularizer=l2(weight_decay)))(o) return ctc_model(x, o)
lstm_shape = (args.num_img_pair - 1, 3, args.img_height, args.img_width) lstm_i1 = Input(shape=lstm_shape) lstm_i2 = Input(shape=lstm_shape) if not args.resume: if args.pre_trained: cnn_model = model.flownetS_to_load(args.img_height, args.img_width) l = cnn_model.layers imgs = layers.concatenate([lstm_i1, lstm_i2], axis=2) x = layers.TimeDistributed(l[1])(imgs) for i in range(2, 27): x = layers.TimeDistributed(l[i])(x) x = layers.TimeDistributed(layers.Flatten())(x) if not args.bidirectional: x = layers.LSTM(1000, return_sequences=True, dropout=args.dropout, recurrent_dropout=args.recurrent_dropout)(x) x = layers.LSTM(1000, return_sequences=True, dropout=args.dropout, recurrent_dropout=args.recurrent_dropout)(x) else: x = layers.Bidirectional( layers.LSTM(1000, return_sequences=True, dropout=args.dropout, recurrent_dropout=args.recurrent_dropout))(x) x = layers.Bidirectional( layers.LSTM(1000, return_sequences=True, dropout=args.dropout,
dataY = numpy.array(dataY) print('dataY shape: ', dataY.shape) #exit() return dataX, dataY time_step = 3 trainX, trainY = create_dataset(X_train, time_step) testX, testY = create_dataset(X_test, time_step) #Creating the RNN and LSTM Neural Networks RNN = Sequential() RNN.add(layers.SimpleRNN(100, input_shape=(time_step, features))) RNN.add(layers.Dense(1)) LSTM = Sequential() LSTM.add(layers.LSTM(512, input_shape=(time_step, features))) LSTM.add(layers.Dense(1)) #Using the mean squared error to determine our training value loss RNN.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy']) LSTM.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy']) #establishing the training history for both the RNN and LSTM history_RNN = RNN.fit(trainX, trainY, validation_data=(testX, testY), epochs=5, batch_size=1) history_LSTM = LSTM.fit(trainX, trainY, validation_data=(testX, testY), epochs=5, batch_size=1) #Plotting our training history for both the LSTM and RNN plt.plot(history_RNN.history['val_loss'], label='val_loss') plt.plot(history_RNN.history['loss'], label='loss') plt.title('RNN Model Loss')
print (x_train) print(x_test) #Takes the vocabulary size in order to create the matrix of the embending layer vocab_size=len(tokenizer.word_index)+1 print(vocab_size) #The data should come as a vectors of the same size, we put 0 at the end of the shorter data x_train=pad_sequences(x_train,padding='post',maxlen=max_ln) x_test=pad_sequences(x_test,padding='post',maxlen=max_ln) #Create the model imput 3 Layers, one is LSTM so our NN is recursive model1 = Sequential() embedding_layer = layers.Embedding(vocab_size, max_words, input_length=max_ln, trainable=False) model1.add(embedding_layer) model1.add(layers.LSTM(128)) model1.add(layers.Dense(1, activation='sigmoid')) model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc']) #Print Model's summary print(model1.summary()) #Fiting and training the model in a 100 epochs, the NN inputs data in batches of 128(Input Layer) history = model1.fit(x_train, y_train, batch_size=128, epochs=100, verbose=1, validation_split=0.2) #Evaluate Model score = model1.evaluate(x_test, y_test, verbose=1) #Print Accuracy and loss print("Test Score:", score[0]) print("Test Accuracy:", score[1])
height = len(cm[0]) for x in range(width): for y in range(height): ax.annotate(str(cm[x][y]), xy=(y, x), horizontalalignment='center', verticalalignment='center', color=getFontColor(cm[x][y])) # add genres as ticks alphabet = mods plt.xticks(range(width), alphabet[:width], rotation=30) plt.yticks(range(height), alphabet[:height]) return plt from keras import layers from keras import Sequential,layers model=Sequential() model.add(layers.LSTM(128,dropout=0.7,return_sequences=True,input_shape=(128,2))) model.add(layers.LSTM(128,dropout=0.7)) model.add(layers.Dense(len(mods),activation="sigmoid")) model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy']) nb_epoch = 80 # number of epochs to train on batch_size = 256 # training batch size ###################################################train the network################################################# history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=nb_epoch, verbose=1, validation_data=(X_test, Y_test) ) # we re-load the best weights once training is finished
def retain(ARGS): """Create the model""" #Define the constant for model saving reshape_size = ARGS.emb_size + ARGS.numeric_size if ARGS.allow_negative: embeddings_constraint = FreezePadding() beta_activation = 'tanh' output_constraint = None else: embeddings_constraint = FreezePadding_Non_Negative() beta_activation = 'sigmoid' output_constraint = non_neg() #Get available gpus , returns empty list if none glist = get_available_gpus() def reshape(data): """Reshape the context vectors to 3D vector""" return K.reshape(x=data, shape=(K.shape(data)[0], 1, reshape_size)) #Code Input codes = L.Input((None, None), name='codes_input') inputs_list = [codes] #Calculate embedding for each code and sum them to a visit level codes_embs_total = L.Embedding( ARGS.num_codes + 1, ARGS.emb_size, name='embedding', embeddings_constraint=embeddings_constraint)(codes) codes_embs = L.Lambda(lambda x: K.sum(x, axis=2))(codes_embs_total) #Numeric input if needed if ARGS.numeric_size: numerics = L.Input((None, ARGS.numeric_size), name='numeric_input') inputs_list.append(numerics) full_embs = L.concatenate([codes_embs, numerics], name='catInp') else: full_embs = codes_embs #Apply dropout on inputs full_embs = L.Dropout(ARGS.dropout_input)(full_embs) #Time input if needed if ARGS.use_time: time = L.Input((None, 1), name='time_input') inputs_list.append(time) time_embs = L.concatenate([full_embs, time], name='catInp2') else: time_embs = full_embs #Setup Layers #This implementation uses Bidirectional LSTM instead of reverse order # (see https://github.com/mp2893/retain/issues/3 for more details) #If training on GPU and Tensorflow use CuDNNLSTM for much faster training if glist: alpha = L.Bidirectional(L.CuDNNLSTM(ARGS.recurrent_size, return_sequences=True), name='alpha') beta = L.Bidirectional(L.CuDNNLSTM(ARGS.recurrent_size, return_sequences=True), name='beta') else: alpha = L.Bidirectional(L.LSTM(ARGS.recurrent_size, return_sequences=True, implementation=2), name='alpha') beta = L.Bidirectional(L.LSTM(ARGS.recurrent_size, return_sequences=True, implementation=2), name='beta') alpha_dense = L.Dense(1, kernel_regularizer=l2(ARGS.l2)) beta_dense = L.Dense(ARGS.emb_size + ARGS.numeric_size, activation=beta_activation, kernel_regularizer=l2(ARGS.l2)) #Compute alpha, visit attention alpha_out = alpha(time_embs) alpha_out = L.TimeDistributed(alpha_dense, name='alpha_dense_0')(alpha_out) alpha_out = L.Softmax(axis=1)(alpha_out) #Compute beta, codes attention beta_out = beta(time_embs) beta_out = L.TimeDistributed(beta_dense, name='beta_dense_0')(beta_out) #Compute context vector based on attentions and embeddings c_t = L.Multiply()([alpha_out, beta_out, full_embs]) c_t = L.Lambda(lambda x: K.sum(x, axis=1))(c_t) #Reshape to 3d vector for consistency between Many to Many and Many to One implementations contexts = L.Lambda(reshape)(c_t) #Make a prediction contexts = L.Dropout(ARGS.dropout_context)(contexts) output_layer = L.Dense(1, activation='sigmoid', name='dOut', kernel_regularizer=l2(ARGS.l2), kernel_constraint=output_constraint) #TimeDistributed is used for consistency # between Many to Many and Many to One implementations output = L.TimeDistributed(output_layer, name='time_distributed_out')(contexts) #Define the model with appropriate inputs model = Model(inputs=inputs_list, outputs=[output]) return model
# 计算val_steps,这是计算需要从val_gen中抽取多少次 val_steps=(valnum-lookback)//batch_size # 查看训练集需要抽取的次数 train_steps=trainnum//batch_size from keras.callbacks import EarlyStopping early_stopping=EarlyStopping(monitor="val_loss",patience=30) from keras.callbacks import ReduceLROnPlateau lr_reduce=ReduceLROnPlateau(monitor="val_loss,factor=0.1,patience=10") #创建模型 model1=models.Sequential() model1.add(layers.Dense(512,activation="relu",input_shape=(None,data.shape[-1]))) model1.add(layers.Dropout(0.5)) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True))) model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5))) model1.add(layers.Dense(512,activation="relu")) model1.add(layers.Dropout(0.5)) model1.add(layers.Dense(8,activation="softmax")) model1.summary() model1.compile(optimizer=optimizers.RMSprop(), loss="categorical_crossentropy", metrics=["acc"])
global_en_mac_tune = 0 global_de_mac_tune = 0 num_iterations = 10 for i in range(num_iterations): print('training iteration:', i + 1) # build model model = models.Sequential() model.add( layers.Embedding(vocab_size, EMBEDDING_DIM, weights=[embedding_matrix], trainable=False, input_length=MAXLEN)) model.add(layers.Bidirectional(layers.LSTM(128))) model.add(layers.Dropout(0.2)) model.add(layers.Dense(64, activation='relu')) model.add(layers.Dense(64, activation='relu')) model.add(layers.Dense(3, activation='softmax')) Adam = optimizers.Adam(learning_rate=0.0001) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc']) print(model.summary()) print(K.eval(model.optimizer.lr)) es = EarlyStopping(monitor='val_loss', mode='auto', min_delta=0, patience=5, restore_best_weights=True,
print(X.shape) print(Y.shape) print('Build model...') words_num = tools.word_num EMBEDDING_SIZE = 128 HIDDEN_LAYER_SIZE = 64 model = Sequential() #model.add(layers.embeddings.Embedding(words_num, 64, input_length=26)) model.add( layers.embeddings.Embedding(words_num, EMBEDDING_SIZE, input_length=input_data_X_size)) model.add(layers.LSTM(HIDDEN_LAYER_SIZE, dropout=0.1, return_sequences=True)) #model.add(layers.Dropout(0.1)) model.add(layers.LSTM(64, return_sequences=True)) #model.add(layers.Dropout(0.1)) model.add(layers.Flatten()) model.add(layers.Dense(2)) #[0, 1] or [1, 0] model.add(layers.Activation('softmax')) #optimizer=keras.optimizers.Adam(lr=0.001) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() model.fit(X, Y, epochs=1, batch_size=64, validation_split=0.05, verbose=2) model.save(dir_path + '/log/qinggan.save.' +
def cnn_2x_siamese(voc_size, max_len, dropout=0.5): # Determine proper input shape #input_shape = _obtain_input_shape(input_shape, # default_size=224, # min_size=197, # data_format=K.image_data_format(), # include_top=include_top) pivot_input = layers.Input(shape=(max_len, ), dtype='int32') statement_input = layers.Input(shape=(max_len, ), dtype='int32') x = layers.Embedding(output_dim=256, input_dim=voc_size, input_length=max_len)(pivot_input) #if input_tensor is None: # img_input = Input(shape=input_shape) # else: # if not K.is_keras_tensor(input_tensor): # img_input = Input(tensor=input_tensor, shape=input_shape) # else: # img_input = input_tensor x = ZeroPadding1D(padding=1)(x) x = Conv1D(64, 7, strides=2, name='conv1')(x) x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = Activation('relu')(x) x = MaxPooling1D(3, strides=2)(x) x = conv_block(x, 3, 32, stage=2, block='a') x = identity_block(x, 3, 32, stage=2, block='b') x = identity_block(x, 3, 32, stage=2, block='c') x = conv_block(x, 3, 64, stage=3, block='a') x = identity_block(x, 3, 64, stage=3, block='b') x = identity_block(x, 1, 64, stage=3, block='c') x = identity_block(x, 3, 64, stage=3, block='d') x = conv_block(x, 3, 128, stage=4, block='a') x = identity_block(x, 3, 128, stage=4, block='b') x = identity_block(x, 3, 128, stage=4, block='c') x = identity_block(x, 3, 128, stage=4, block='d') x = identity_block(x, 3, 128, stage=4, block='e') x = identity_block(x, 3, 128, stage=4, block='f') x = conv_block(x, 3, 256, stage=5, block='a') x = identity_block(x, 3, 256, stage=5, block='b') x = identity_block(x, 3, 256, stage=5, block='c') x = AveragePooling1D(7, name='avg_pool')(x) embedded_pivot = layers.LSTM(256)(x) encoder_model = Model(pivot_input, embedded_pivot) embedded_statement = encoder_model(statement_input) concat = layers.merge([embedded_pivot, embedded_statement], mode='concat') x = layers.Dense(256, activation='relu')(concat) x = layers.Dropout(dropout)(x) prediction = layers.Dense(1, activation='sigmoid')(x) model = Model([pivot_input, statement_input], prediction) return model
def __build_net__(self): # 构建模型/网络 # input inputs_data = layers.Input(shape=( 50, 75, )) inputs_reward = layers.Input(shape=(1, )) inputs_target = layers.Input(shape=(50, )) # net x = layers.LSTM(units=128, batch_input_shape=(None, 50, 75), return_sequences=0)(inputs_data) x = layers.Dense(128, activation='relu')(x) predictions = layers.Dense(50, activation='sigmoid')(x) # model_choose model_choose = Model(inputs=inputs_data, outputs=predictions) # loss def lambda_fuc_1(inputx): predictions, inputs_target, inputs_reward = inputx result = K.mean( K.square(predictions - inputs_target)) * (inputs_reward) # return K.flatten(inputs_reward+1) return result output_loss = layers.Lambda(function=lambda_fuc_1)( [predictions, inputs_target, inputs_reward]) # output_loss = layers.Lambda(lambda x: ((x[0]-x[1])**2)*x[2])([predictions, inputs_target, inputs_reward]) # model_for_train model_for_train = Model( inputs=[inputs_data, inputs_target, inputs_reward], outputs=[output_loss, predictions]) # summary model_choose.summary() model_for_train.summary() plot_model(model_choose, to_file="model_image/choose_model.png", show_layer_names=True, show_shapes=True) plot_model(model_for_train, to_file="model_image/choose_model_for_trian.png", show_layer_names=True, show_shapes=True) # 编译步骤 def lambda_x(y_true, y_pred): return y_pred def lambda_y(y_true, y_pred): result = K.zeros_like(y_pred) return result losses = [ lambda_x, # loss is computed in Lambda layer lambda_y # we only include this for the metrics ] model_choose.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model_for_train.compile( # optimizer='rmsprop', optimizer='rmsprop', loss=losses, ) return model_choose, model_for_train
#################### train and test split x_train_lstm_daily = samples_lstm_daily[:4000] x_test_lstm_daily = samples_lstm_daily[4000:] y_train = labels[:4000] y_test = labels[4000:] print(x_train_lstm_daily.shape,y_train.shape) print(x_test_lstm_daily.shape,y_test.shape) plt.figure(figsize=(20,10)) plt.plot(y_test,'r') ################################################ Model: daily Block - Multi-STGCnet input130 = Input(shape=(pre,1), dtype='float') input131 = layers.LSTM(35,return_sequences=True)(input130) input132 = layers.LSTM(12)(input131) output13 = layers.Dense(1,activation='relu')(input132) model = models.Model(inputs=[input130],outputs=[output13]) model.summary() model.compile(optimizer='rmsprop',loss='mae',metrics=['mae','mse','mape']) callbacks_list = [ keras.callbacks.ModelCheckpoint(filepath='lstm_daily.h5', monitor='val_loss', save_best_only=True,) ################################################ Model training epochs = 1000 H = model.fit([x_train_lstm_daily], y_train,callbacks=callbacks_list,batch_size=interval_batch,epochs=epochs,validation_data=([x_test_lstm_daily],y_test)) ################################################ Loss
from keras import Sequential max_features = 10000 maxlen = 500 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) x_train = [x[::-1] for x in x_train] x_test = [x[::-1] for x in x_test] x_train = sequence.pad_sequences(x_train, maxlen=maxlen) x_test = sequence.pad_sequences(x_test, maxlen=maxlen) models = Sequential() model.add(layers.Embedding(max_features, 128)) model.add(layers.LSTM(32)) model.add(layers.Dense(1, activation='sigmoid')) model.compile(optimizer='rmsprop', loss='binarycrossentropy', metrics=['acc']) accuracy = model.fit(x_train, y_train, epochs=10, batch_size=128, validation_split=0.2) model = Sequential() model.add(layers.Embedding(max_features, 32)) model.add(layers.Bidirectional((layers.LSTM(32)))) model.add(layers.Dense(1, activation='sigmoid')) model.compile(optimizer='rmsprop', loss='binarycrossentropy', metrics=['acc']) accuracy = model.fit(x_train,
if layer.__class__.__name__ == 'GRU': W = [np.split(w, 3, axis=-1) for w in weights] return sum(map(list, zip(*W)), []) elif layer.__class__.__name__ in ('LSTM', 'ConvLSTM2D'): W = [np.split(w, 4, axis=-1) for w in weights] for w in W: w[2], w[1] = w[1], w[2] return sum(map(list, zip(*W)), []) elif layer.__class__.__name__ == 'Conv2DTranspose': return [np.transpose(weights[0], (2, 3, 0, 1)), weights[1]] return weights @pytest.mark.parametrize("layer", [ layers.GRU(2, input_shape=[3, 5]), layers.LSTM(2, input_shape=[3, 5]), layers.ConvLSTM2D( 5, (3, 3), input_shape=[6, 6, 6, 6], data_format='channels_first'), ], ids=['GRU', 'LSTM', 'ConvLSTM2D']) def test_preprocess_weights_for_loading(layer): # A model is needed to initialize weights. _ = Sequential([layer]) weights1 = layer.get_weights() weights2 = saving.preprocess_weights_for_loading( layer, convert_weights(layer, weights1), original_keras_version='1') assert all([np.allclose(x, y, 1e-5) for (x, y) in zip(weights1, weights2)]) @pytest.mark.parametrize("layer", [ layers.Conv2D(2, (3, 3), input_shape=[5, 5, 3]),
def test_sequential_regression(): # start with a basic example of using a Sequential model # inside the functional API seq = models.Sequential() seq.add(layers.Dense(10, input_shape=(10, ))) x = layers.Input(shape=(10, )) y = seq(x) model = models.Model(x, y) model.compile('rmsprop', 'mse') weights = model.get_weights() # test serialization config = model.get_config() model = models.Model.from_config(config) model.compile('rmsprop', 'mse') model.set_weights(weights) # more advanced model with multiple branches branch_1 = models.Sequential(name='branch_1') branch_1.add( layers.Embedding(input_dim=100, output_dim=10, input_length=2, name='embed_1')) branch_1.add(layers.LSTM(32, name='lstm_1')) branch_2 = models.Sequential(name='branch_2') branch_2.add(layers.Dense(32, input_shape=(8, ), name='dense_2')) branch_3 = models.Sequential(name='branch_3') branch_3.add(layers.Dense(32, input_shape=(6, ), name='dense_3')) branch_1_2 = models.Sequential( [legacy_layers.Merge([branch_1, branch_2], mode='concat')], name='branch_1_2') branch_1_2.add(layers.Dense(16, name='dense_1_2-0')) # test whether impromptu input_shape breaks the model branch_1_2.add(layers.Dense(16, input_shape=(16, ), name='dense_1_2-1')) model = models.Sequential( [legacy_layers.Merge([branch_1_2, branch_3], mode='concat')], name='final') model.add(layers.Dense(16, name='dense_final')) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() x = (100 * np.random.random((100, 2))).astype('int32') y = np.random.random((100, 8)) z = np.random.random((100, 6)) labels = np.random.random((100, 16)) model.fit([x, y, z], labels, epochs=1) # test if Sequential can be called in the functional API a = layers.Input(shape=(2, ), dtype='int32') b = layers.Input(shape=(8, )) c = layers.Input(shape=(6, )) o = model([a, b, c]) outer_model = models.Model([a, b, c], o) outer_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) outer_model.fit([x, y, z], labels, epochs=1) # test serialization config = outer_model.get_config() outer_model = models.Model.from_config(config) outer_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) outer_model.fit([x, y, z], labels, epochs=1)
model2 = keras.Sequential([ layers.Dense(64, input_shape=(32, )), layers.Dense(32, activation=tf.nn.relu), layers.Dropout(0.10), layers.Dense(32, activation=tf.nn.relu), layers.Dense(32, activation=tf.nn.relu), layers.Dense(12, activation=tf.nn.relu), layers.Dense(12, activation=tf.nn.relu), layers.Dense(1) ]) xtrcnn = np.reshape(xtr.values, (xtr.values.shape[0], 1, xtr.values.shape[1])) ytrcnn = np.reshape(ytr.values, (ytr.shape[0], 1)) model = keras.Sequential([ layers.LSTM(64, input_shape=(xtrcnn.shape[1], xtrcnn.shape[2])), layers.Dense(64, activation=tf.nn.relu), layers.Dense(32, activation=tf.nn.relu), layers.Dropout(0.10), layers.Dense(32, activation=tf.nn.relu), layers.Dense(32, activation=tf.nn.relu), layers.Dense(12, activation=tf.nn.relu), layers.Dense(12, activation=tf.nn.relu), layers.Dense(1) ]) model2.compile(loss='mean_squared_error', optimizer='adam', metrics=['mean_absolute_error', 'mean_squared_error']) model2.fit(tr_x.values, tr_y, epochs=5)
'layer_norm': self.layer_norm, 'mi': self.mi, 'zoneout_h': self.zoneout_h, 'zoneout_c': self.zoneout_c } base_config = super(LSTM, self).get_config() return dict(list(base_config.items()) + list(config.items())) # keras LSTM: OK b = Bidirectional( keras_layers.LSTM(256, return_sequences=True, kernel_regularizer=l2(1e-4), recurrent_regularizer=l2(1e-4), dropout=0, recurrent_dropout=0, activation='tanh')) # customizer LSTM: Error test = Bidirectional( LSTM(256, return_sequences=True, kernel_regularizer=l2(1e-4), recurrent_regularizer=l2(1e-4), dropout=0, recurrent_dropout=0, zoneout_c=0, zoneout_h=0, mi=None,
from keras import optimizers out = [] for k in [2, 6, 12, 24, 36, 48]: for l in [2, 4, 8, 16, 32, 64]: print('running at %d and %d' % (k, l)) sample, target = create_datasets(df, k) ix = int(np.ceil(0.8 * len(df))) sample_train, target_train = sample[0:ix], target[0:ix] sample_test, target_test = sample[ix:], target[ix:] model = models.Sequential() model.add( layers.LSTM(l, input_shape=(sample_train.shape[1], sample_train.shape[-1]))) model.add(layers.Dense(1, activation="sigmoid")) model.compile(optimizer=optimizers.RMSprop(), loss="binary_crossentropy", metrics=['mae']) hist = model.fit(sample_train, target_train, epochs=100, batch_size=1, verbose=2, validation_data=(sample_test, target_test)) out.append([ k, l, hist.history['mean_absolute_error'][99], hist.history['val_mean_absolute_error'][99]
batch_size=batch_size) all_gen = generator(all_data, lookback=lookback, step=step, min_index=0, max_index=len(all_data), batch_size=batch_size) train_steps = (bound - lookback) // batch_size val_steps = (len(all_data) - bound + 1 - lookback) // batch_size test_steps = (len(all_data2) - lookback) // batch_size all_steps = (len(all_data) - lookback) // batch_size # dropout LSTM model = models.Sequential() model.add(layers.LSTM(64, input_shape=(None, all_data.shape[1] - 11))) model.add(layers.Dense(1, activation='sigmoid')) model.compile(optimizer=RMSprop(), loss='binary_crossentropy', metrics=['acc']) history = model.fit_generator(train_gen, steps_per_epoch=train_steps, epochs=20, validation_data=val_gen, validation_steps=val_steps, callbacks=callbacks_list, verbose=0) # generate trainset prediction results model = models.load_model(model_path) preds = model.predict_generator(all_gen, steps=all_steps)[:, 0]
#Set up the chars to be encoded char_indices = dict((char, chars.index(char)) for char in chars) #One hot encode the characters into binary arryas print('Vectorization...') x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool) y = np.zeros((len(sentences), len(chars)), dtype=np.bool) for i, sentence in enumerate(sentences): for t, char in enumerate(sentence): x[i, t, char_indices[char]] = 1 y[i, char_indices[next_chars[i]]] = 1 #Very simple model, standard/suitable configuration with only two layers, as we're only dealing with text model = keras.models.Sequential() #LSTM is our secret weapon here - as our iterations become more accurate, LSTM will preserve the good patterns and discard the noise model.add(layers.LSTM(128, input_shape=(maxlen, len(chars)))) model.add(layers.Dense(len(chars), activation='softmax')) optimizer = keras.optimizers.RMSprop(lr=0.01) model.compile(loss='categorical_crossentropy', optimizer=optimizer) #This 'sample' function is the core part of this program #It sets up a probability distribution for which character to select after another, with the temperature variable being used as a #weighting for entropy i.e. the higher the temp the less reliable and typical the character choice #So leveragin this function, our program will try its best to crack the statistical code that lies behind Nietzsche's unique style def sample(preds, temperature=1.0): preds = np.asarray(preds).astype('float64') preds = np.log(preds) / temperature exp_preds = np.exp(preds)
# vectorizing data Xitrain, Xqtrain, Ytrain = vectorize(train_data, word2idx, word2idx_answers, max_len_instance, max_len_question) Xitest, Xqtest, Ytest = vectorize(test_data, word2idx, word2idx_answers, max_len_instance, max_len_question) # params epochs = 40 dropout_rate = 0.05 embedding_size_text = 112 embedding_size_question = 96 latent_size_text = 142 latent_size_question = 128 # model text_input = Input(shape=(max_len_instance,)) embedded_text = layers.Embedding(vocabulary_size, embedding_size_text)(text_input) encoded_text = layers.LSTM(latent_size_text)(embedded_text) encoded_text = Dropout(dropout_rate)(encoded_text) question_input = Input(shape=(max_len_question,)) embedded_question = layers.Embedding(vocabulary_size, embedding_size_question)(question_input) encoded_question = layers.LSTM(latent_size_question)(embedded_question) encoded_question = Dropout(dropout_rate)(encoded_question) concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1) answer = layers.Dense(vocabulary_size_answers, activation='softmax')(concatenated) answer = Dropout(dropout_rate)(answer) model = Model([text_input, question_input], answer) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc']) # training