class_weight = {0: 1., 1: 300.} #%% LSTM model = Sequential() model.add( Conv2D(filters=256, kernel_size=(7, 2), activation='tanh', input_shape=(x_train.shape[1], x_train.shape[2], 1), padding='valid')) model.add(Dropout(0.3)) model.add( Conv2D(filters=256, kernel_size=(7, 1), activation='tanh', padding='same')) model.add(Dropout(0.2)) model.add(TimeDistributed(Flatten())) model.add(GRU(units=256, return_sequences=True)) model.add(GRU(units=256, return_sequences=True)) model.add(Flatten()) model.add(Dense(units=512, activation='tanh')) model.add(Dropout(0.4)) model.add(Dense(units=128, activation='tanh')) model.add(Dense(units=1, activation='sigmoid')) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) model_history = model.fit(x_train, y_train, batch_size=1024, epochs=50,
def build_discriminator(): # one aproch # https://towardsdatascience.com/an-approach-towards-convolutional-recurrent-neural-networks-f54cbeecd4a6 # vary setings shape_in_x = int(320), int(8), int(1) shape_in_y = int(720), int(1280), int(3) shape_out = int(8134), int(120) dropoutrate = 0.3 x_start = Input(shape=(shape_in_x)) x = x_start for _i, _cnt in enumerate((2, 2)): x = Conv2D(filters = 100, kernel_size=(2, 2), padding='same',)(x) x = BatchNormalization(axis=1)(x) #x = Activation('relu')(x) x = LeakyReLU()(x) #x = MaxPooling2D(pool_size=(2,2), dim_ordering="th" )(x) x = MaxPooling2D(pool_size=2)(x) x = Dropout(dropoutrate)(x) x = Permute((2, 1, 3))(x) x = Reshape((1, 16000))(x) #out = Activation('sigmoid', name='strong_out')(x) #audio_context = Model(inputs=x_start, outputs=out) #audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy']) #audio_context.summary() y_start = Input(shape=(shape_in_y)) y = y_start for _i, _cnt in enumerate((2, 2)): y = Conv2D(filters = 100, kernel_size=(2, 2), padding='same',)(y) y = BatchNormalization(axis=1)(y) #y = Activation('relu')(y) y = LeakyReLU()(y) #y = MayPooling2D(pool_size=(2,2), dim_ordering="th" )(y) y = MaxPooling2D(pool_size=2)(y) y = Dropout(dropoutrate)(y) y = Permute((2, 1, 3))(y) y = Reshape((1, 5760000))(y) #out = Activation('sigmoid', name='strong_out')(y) #audio_context = Model(inputs=y_start, outputs=out) #audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy']) #audio_context.summary() z = Concatenate()([x, y]) #out = Activation('sigmoid', name='strong_out')(z) #audio_context = Model(inputs=[x_start,y_start], outputs=out) #audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy']) #audio_context.summary() # The Gru/recurrent portion # Get some knowledge # http://colah.github.io/posts/2015-08-Understanding-LSTMs/ for r in (10,10): z = Bidirectional( GRU(r, activation='tanh', dropout=dropoutrate, recurrent_dropout=dropoutrate, return_sequences=True), merge_mode='concat')(z) for f in ((2,2)): z = TimeDistributed(Dense(f))(z) #z = Dropout(dropoutrate)(z) #z = TimeDistributed(Dense(880))(z) # arbitrary reshape may be a problem z = Flatten()(z) z = Dropout(dropoutrate)(z) z = Dense(1,activation = 'sigmoid')(z) #out = Activation('sigmoid', name='strong_out')(z) descrim = Model(inputs=[x_start, y_start], outputs=z) descrim.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy']) #descrim.summary() #ce = audio_context return descrim
import numpy as np x = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]]) y = np.array([4, 5, 6, 7]) print('x.shape : ', x.shape) #(4,3) print('y.shape : ', y.shape) #(4,) x = x.reshape(4, 3, 1) #2. 모델구성 from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, LSTM, GRU model = Sequential() model.add(GRU(10, activation='relu', input_shape=(3, 1))) #x변경 됐음 model.add(Dense(20)) model.add(Dense(10)) model.add(Dense(1)) #LSTM있는 곳에 디폴트가 탄젠트인것, output은 linear임 # 여기까지는 회귀값이다. 분류값 아님 #model.summary() #3. 컴파일 , 훈련 model.compile(loss='mse', optimizer='adam') model.fit(x, y, epochs=100, batch_size=1) #4. 평가, 예측 loss = model.evaluate(x, y) print(loss) x_pred = np.array([5, 6, 7]) #(3,) 행은 하나 -> (1,3,1)
def build_attention_model(self): if self.cell_type == "RNN": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = SimpleRNN( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state = encoder(encoder_inputs) if i == 1: encoder_first_outputs= encoder_outputs encoder_states = [state] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = SimpleRNN( self.latentDim, return_sequences=True, return_state=True, dropout=self.dropout, ) decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states) if i == self.numDecoders: decoder_first_outputs = decoder_outputs attention_layer = AttentionLayer(name='attention_layer') attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs]) decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out]) # dense hidden = Dense(self.hidden, activation="relu") hidden_time = TimeDistributed(hidden, name='time_distributed_layer') hidden_outputs = hidden(decoder_concat_input) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model elif self.cell_type == "LSTM": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = LSTM( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state_h, state_c = encoder(encoder_outputs) if i == 1: encoder_first_outputs= encoder_outputs encoder_states = [state_h, state_c] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = LSTM( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) decoder_outputs, _, _ = decoder( decoder_outputs, initial_state=encoder_states ) if i == self.numDecoders: decoder_first_outputs = decoder_outputs attention_layer = AttentionLayer(name='attention_layer') attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs]) decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out]) # dense hidden = Dense(self.hidden, activation="relu") hidden_time = TimeDistributed(hidden, name='time_distributed_layer') hidden_outputs = hidden(decoder_concat_input) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model elif self.cell_type == "GRU": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = GRU( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state = encoder(encoder_inputs) if i == 1: encoder_first_outputs= encoder_outputs encoder_states = [state] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = GRU( self.latentDim, return_sequences=True, return_state=True, dropout=self.dropout, ) decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states) if i == self.numDecoders: decoder_first_outputs = decoder_outputs attention_layer = AttentionLayer(name='attention_layer') attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs]) decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out]) # dense hidden = Dense(self.hidden, activation="relu") hidden_time = TimeDistributed(hidden, name='time_distributed_layer') hidden_outputs = hidden(decoder_concat_input) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model
dencoder_seq = dencoder_tokenizer.texts_to_sequences(eng) print(dencoder_seq[1]) dencoder_max_len = max([len(i) for i in dencoder_seq]) dencoder_pad = np.array(pad_sequences(dencoder_seq, maxlen=dencoder_max_len + 1, padding='post', truncating='post'), dtype='float') dencoder_input_data = dencoder_pad[:, :-1] dencoder_output_data = dencoder_pad[:, 1:] dencoder_tokens = dict([(w, q) for q, w in dencoder_words.items()]) encoder_input = Input(shape=(None, )) encoder_embedding = Embedding(total_words, 128)(encoder_input) encoder_gru1 = GRU(128, return_sequences=True)(encoder_embedding) encoder_gru2 = GRU(128, return_sequences=True)(encoder_gru1) encoder_gru3 = GRU(128, return_sequences=False)(encoder_gru2) # encoder_model=Sequential() # encoder_model.add(Embedding(total_words, 128, input_length=encoder_max_len)) # encoder_model.add(Bidirectional(GRU(128,return_sequences = True))) # encoder_model.add(Bidirectional(GRU(128,return_sequences = True))) # encoder_outputs, state_h=encoder_model.add(Bidirectional(GRU(128,return_sequences = False,return_state=True))) # encoder_outputs, state_h=encoder_model.output dencoder_initial_state = Input(shape=(128, )) dencoder_input = Input(shape=(None, )) dencoder_embedding = Embedding(total_words, 128)(dencoder_input) dencoder_gru1 = GRU(128, return_sequences=True)(dencoder_embedding, initial_state=encoder_gru3) dencoder_gru2 = GRU(128, return_sequences=True)(dencoder_gru1,
X = roll[:seq_len].reshape(seq_len) / float(len(alphabet)) y = utils.to_categorical(roll[seq_len % len(alphabet)], len(alphabet)) data["X"].append(X) data["y"].append(y) X = pad_sequences(data["X"], maxlen=len(alphabet), value=mask_value, dtype=np.float32) X = np.reshape(X, (X.shape[0], X.shape[1], 1)) y = np.array(data["y"]) y = np.reshape(y, (y.shape[0], y.shape[2])) model = Sequential() model.add(Masking(mask_value=mask_value, input_shape=(len(alphabet), 1))) model.add(GRU(32)) model.add(Dense(len(alphabet), activation="softmax")) model.compile(loss="categorical_crossentropy", optimizer="nadam", metrics=["accuracy"]) print(model.summary()) class EvaluationCallback(Callback): def __init__(self, metric): if metric == "loss": self.metric = "loss" else: self.metric = "accuracy" print("===> Metric is", metric)
dataset = dataset.reshape(40, 10, 2) label = [] for i in range(5, 445): if 94 < i <105 or 194 < i <205 or 294 < i <305 or 394 < i <405: continue y_data = [dataset_x[i]/100, dataset_y[i]/100] label.append(y_data) label = np.array(label, dtype=float) label = label.reshape(40, 20) # split data and labels into train and test x_train, x_test, y_train, y_test = train_test_split(dataset, label, test_size=0.2, random_state=4) print y_test.shape model =Sequential() model.add(GRU(units=50, return_sequences=True, input_shape = (x_train.shape[1],2))) model.add(GRU(units=50)) model.add(Dense(20)) model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy']) model.summary() history = model.fit(x_train, y_train, epochs=500, validation_data=(x_test, y_test)) results = model.predict(x_test) results = results.reshape(results.shape[0],10,2) y_test = y_test.reshape(8, 10, 2) results = results.reshape(8, 10, 2) a_axis=[] b_axis=[] a_r_axis = [] b_r_axis = [] for i in range(8):
def _build(self): self.bigru = Bidirectional( GRU(cfg.HIDDEN_DIM // 2, return_sequences=True, return_state=True))
vocab_size = len(tokenizer_obj.word_index) + 1 EMBEDDING_DIM = 100 max_length = max([len(s.split()) for s in x_all]) x_train_tokens = tokenizer_obj.texts_to_sequences(x_train) x_test_tokens = tokenizer_obj.texts_to_sequences(x_test) x_train_pad = pad_sequences(x_train_tokens, maxlen=max_length, padding='post') x_test_pad = pad_sequences(x_test_tokens, maxlen=max_length, padding='post') rnn_model = Sequential() rnn_model.add(Embedding(vocab_size, EMBEDDING_DIM, input_length=max_length)) rnn_model.add(GRU(units=64, dropout=0.1, recurrent_dropout=0.1)) rnn_model.add(BatchNormalization()) rnn_model.add(Dense(1, activation='sigmoid')) _optimizer = optimizers.RMSprop(lr=0.01) rnn_model.compile(loss='binary_crossentropy', optimizer=_optimizer, metrics=['accuracy']) print(rnn_model.summary()) print('Started training the model...') early_stopping = tensorflow.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10) history = rnn_model.fit(x_train_pad, y_train, batch_size=32, epochs=15, validation_data=(x_test_pad, y_test), callbacks=[early_stopping], verbose=2)
dataf = np.reshape(Train_data, [315*1000, 64]) rng_state = np.random.get_state() np.random.shuffle(dataf) np.random.set_state(rng_state) np.random.shuffle(labelsf) labels = np.reshape(labelsf, [315, 1000, 1]) Train_data = np.reshape(dataf, [315, 1000, 64]) X_train, X_test = train_test_split(Train_data,test_size=0.2) y_train, y_test = train_test_split(labels,test_size=0.2) lab = np.reshape(labels, [315*1000,]) class_weights = class_weight.compute_class_weight('balanced', np.unique(lab), lab) training_in_shape = Train_data.shape[1:] training_in = Input(shape = training_in_shape) Var = GRU(1240, return_sequences=True, stateful = False)(training_in) training_pred = Dense(3, activation = 'softmax')(Var) training_model = Model(inputs = training_in, outputs = training_pred) training_model.compile(loss = keras.losses.SparseCategoricalCrossentropy(), optimizer = 'adam', metrics = ['accuracy']) training_model.summary() results = training_model.fit(Train_data, labels, batch_size = 4, epochs = 20, validation_split = 0.2) streaming_in = Input(batch_shape=(1,None,64)) ## stateful ==> needs batch_shape specified foo = GRU(1240, return_sequences=False, stateful=True )(streaming_in) streaming_pred = Dense(3, activation = 'softmax')(foo)
def bert_tensorflow_test(X_train, X_test, Y_train, Y_test): # Model model = Sequential() model.add( Masking(mask_value=0., input_shape=(MAX_SEQUENCE_LEN, VECTOR_DIM))) #forward_layer = LSTM(200, return_sequences=True) forward_layer = GRU(10, return_sequences=False, dropout=0.5) #backward_layer = LSTM(200, activation='relu', return_sequences=True, backward_layer = GRU(10, return_sequences=False, dropout=0.5, go_backwards=True) model.add( Bidirectional(forward_layer, backward_layer=backward_layer, input_shape=(MAX_SEQUENCE_LEN, VECTOR_DIM))) #model.add(TimeDistributed(Dense(NUM_CLASSES))) # Remove TimeDistributed() so that predictions are now made for the entire sentence model.add(Dense(NUM_CLASSES)) model.add(Activation('softmax')) #print('preds shape', model.predict(X_train[:3]).shape) #print('Y_train shape', Y_train[:3].shape) #print(list(Y_train[:3])) classes = [] for y in Y_train: cls = np.argmax(y) classes.append(cls) print(Counter(classes)) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) print('compiled model') model.fit(X_train, Y_train, batch_size=8, epochs=10) #, validation_split=0.1) print('fit model') eval = model.evaluate(X_test, Y_test, batch_size=8) #print('X_test[0]') #print(X_test[0]) #print(X_train[0]) preds = model.predict_proba(X_test, verbose=1, batch_size=8) print(preds) num_correct = 0 num_incorrect = 0 TP = 0 TN = 0 FP = 0 FN = 0 # idiomatic = 2, non-idiomatic = 3 with open('preds_out_temp.txt', 'w') as tempoutf: for pred, y in zip(preds, Y_test): if np.argmax(y) == 2 or np.argmax(y) == 3: if np.argmax(y) == np.argmax(pred): num_correct += 1 else: num_incorrect += 1 if np.argmax(pred) == 2 and np.argmax(y) == 2: TP += 1 if np.argmax(pred) == 3 and np.argmax(y) == 3: TN += 1 if np.argmax(pred) == 2 and np.argmax(y) == 3: FP += 1 if np.argmax(pred) == 3 and np.argmax(y) == 2: FN += 1 custom_accuracy = num_correct / (num_correct + num_incorrect) print('custom accuracy is', num_correct / (num_correct + num_incorrect)) for y in Y_test: cls = np.argmax(y) classes.append(cls) class_nums = Counter(classes) print(class_nums) default_acc = class_nums[2] / (class_nums[2] + class_nums[3]) print('default accuracy is', default_acc, 'or', 1 - default_acc) return eval, custom_accuracy, default_acc, [TP, TN, FP, FN]
labels = np.array(temp) # Divide the data into train, val and test data x_train = data[:20000] x_val = data[20000:24000] x_test = data[24000:30000] x_train_img = image_data[:20000] x_val_img = image_data[20000:24000] x_test_img = image_data[24000:30000] y_train = labels[:20000] y_val = labels[20000:24000] y_test = labels[24000:30000] # Build NwQM-w/oT model page_input = Input(shape=(MAX_SECS, EMBED_DIM), dtype='float32') l_lstm_sec = Bidirectional(GRU(100, return_sequences=True))(page_input) l_att_sec = AttLayer(100)(l_lstm_sec) img_inp = Input(shape=(2048, )) img_rep = Dense(200, activation='relu')(img_inp) norm1 = norm(l_att_sec - img_rep, ord=2, keepdims=True, axis=-1) final_rep = concatenate([l_att_sec, img_rep, norm1], axis=-1) preds = Dense(6, activation='softmax')(final_rep) model = Model([page_input, img_inp], preds) optim = Adam(learning_rate=args.learning_rate, beta_1=0.9, beta_2=0.999, amsgrad=False)
def get_test_model_gru_stateful_optional(stateful): """Returns a test model for Gated Recurrent Unit (GRU) layers.""" input_shapes = [(17, 4), (1, 10)] stateful_batch_size = 1 inputs = [ Input(batch_shape=(stateful_batch_size, ) + s) for s in input_shapes ] outputs = [] for inp in inputs: gru_sequences = GRU(stateful=stateful, units=8, recurrent_activation='relu', reset_after=True, return_sequences=True, use_bias=True)(inp) gru_regular = GRU(stateful=stateful, units=3, recurrent_activation='sigmoid', reset_after=True, return_sequences=False, use_bias=False)(gru_sequences) outputs.append(gru_regular) gru_bidi_sequences = Bidirectional( GRU(stateful=stateful, units=4, recurrent_activation='hard_sigmoid', reset_after=False, return_sequences=True, use_bias=True))(inp) gru_bidi = Bidirectional( GRU(stateful=stateful, units=6, recurrent_activation='sigmoid', reset_after=True, return_sequences=False, use_bias=False))(gru_bidi_sequences) outputs.append(gru_bidi) gru_gpu_regular = GRU(stateful=stateful, units=3, activation='tanh', recurrent_activation='sigmoid', reset_after=True, use_bias=True)(inp) gru_gpu_bidi = Bidirectional( GRU(stateful=stateful, units=3, activation='tanh', recurrent_activation='sigmoid', reset_after=True, use_bias=True))(inp) outputs.append(gru_gpu_regular) outputs.append(gru_gpu_bidi) model = Model(inputs=inputs, outputs=outputs, name='test_model_gru') model.compile(loss='mse', optimizer='nadam') # fit to dummy data training_data_size = stateful_batch_size data_in = generate_input_data(training_data_size, input_shapes) initial_data_out = model.predict(data_in) data_out = generate_output_data(training_data_size, initial_data_out) model.fit(data_in, data_out, batch_size=stateful_batch_size, epochs=10) return model
test_gen = generator(df_values, lookback=LOOKBACK, delay=DELAY, min_index=test_min_i, max_index=test_max_i, batch_size=BATCH_SIZE, step=STEP, shuffle=False) # Instantiate Model ################### clear_session() model3 = Sequential() model3.add( GRU(32, dropout=0.2, recurrent_dropout=0.2, input_shape=(None, df_values.shape[-1]))) model3.add(Dense(1)) model3.compile( optimizer=RMSprop(), loss='mae', ) print(model3.summary()) # Train ####### history3 = model3.fit(train_gen, steps_per_epoch=500, epochs=40, validation_data=val_gen, validation_steps=val_steps)
def flor(input_size, output_size, learning_rate=5e-4): """ Gated Convolucional Recurrent Neural Network by Flor et al. """ input_data = Input(name="input", shape=input_size) cnn = Conv2D(filters=16, kernel_size=(3, 3), strides=(2, 2), padding="same")(input_data) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = FullGatedConv2D(filters=16, kernel_size=(3, 3), padding="same")(cnn) cnn = Conv2D(filters=32, kernel_size=(3, 3), strides=(1, 2), padding="same")(cnn) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = FullGatedConv2D(filters=32, kernel_size=(3, 3), padding="same")(cnn) cnn = Conv2D(filters=40, kernel_size=(2, 4), strides=(2, 2), padding="same")(cnn) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = FullGatedConv2D(filters=40, kernel_size=(3, 3), padding="same", kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn) cnn = Dropout(rate=0.2)(cnn) cnn = Conv2D(filters=48, kernel_size=(3, 3), strides=(1, 2), padding="same")(cnn) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = FullGatedConv2D(filters=48, kernel_size=(3, 3), padding="same", kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn) cnn = Dropout(rate=0.2)(cnn) cnn = Conv2D(filters=56, kernel_size=(2, 4), strides=(2, 2), padding="same")(cnn) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = FullGatedConv2D(filters=56, kernel_size=(3, 3), padding="same", kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn) cnn = Dropout(rate=0.2)(cnn) cnn = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding="same")(cnn) cnn = PReLU(shared_axes=[1, 2])(cnn) cnn = BatchNormalization(renorm=True)(cnn) cnn = MaxPooling2D(pool_size=(1, 2), strides=(1, 2), padding="valid")(cnn) shape = cnn.get_shape() bgru = Reshape((shape[1], shape[2] * shape[3]))(cnn) bgru = Bidirectional(GRU(units=128, return_sequences=True, dropout=0.5))(bgru) bgru = TimeDistributed(Dense(units=128))(bgru) bgru = Bidirectional(GRU(units=128, return_sequences=True, dropout=0.5))(bgru) output_data = TimeDistributed( Dense(units=output_size, activation="softmax"))(bgru) optimizer = RMSprop(learning_rate=learning_rate) return (input_data, output_data, optimizer)
def DMN(n_answer, embedding, mask_zero=True, trainable=True): context_shape = (MAX_CONTEXT_LENGTH, ) question_shape = (MAX_QUESTION_LENGTH, ) answer_shape = (MAX_ANSWER_LENGTH, ) in_context = Input(shape=context_shape, dtype='int32') in_question = Input(shape=question_shape, dtype='int32') in_answer = Input(shape=answer_shape, dtype='int32') embedding_layer = PretrainedEmbedding(embeddings=embedding, mask_zero=mask_zero, rate=DROPOUT_RATE) context, context_mask = embedding_layer( in_context), embedding_layer.compute_mask(in_context) question, question_mask = embedding_layer( in_question), embedding_layer.compute_mask(in_question) context, c_lst = GRU(units=INPUT_LAYER_UNIT, return_sequences=True, return_state=True)(context, mask=context_mask) question = GRU(units=INPUT_LAYER_UNIT, return_sequences=False, return_state=False)(question, mask=question_mask) # question = tf.expand_dims(question, axis=1) initial_states = [question, question, question] # TODO: this looks problematic. Maybe we should not pass m from the beginning to EPISODIC CELL? m1, states = EpisodicModule(units=EPISODIC_LAYER_UNITS, attention_layer_units=ATTENTION_LAYER_UNITS, reg_scale=REG_SCALE, trainable=trainable, return_sequences=False, return_state=True, zero_output_for_mask=True, unroll=True)(context, initial_state=initial_states, mask=context_mask) # m1 = tf.expand_dims(m1, axis=1) initial_states = [states, m1, question] m2, states2 = EpisodicModule(units=EPISODIC_LAYER_UNITS, attention_layer_units=ATTENTION_LAYER_UNITS, reg_scale=REG_SCALE, trainable=trainable, return_sequences=False, return_state=True, zero_output_for_mask=True, unroll=True)(context, initial_state=initial_states, mask=context_mask) # m2 = tf.expand_dims(m2, axis=1) initial_states = [states2, m2, question] m = EpisodicModule(units=EPISODIC_LAYER_UNITS, attention_layer_units=ATTENTION_LAYER_UNITS, reg_scale=REG_SCALE, trainable=trainable, return_sequences=False, return_state=False, zero_output_for_mask=True, unroll=True)(context, initial_state=initial_states, mask=context_mask) input = tf.concat([m, question], axis=-1) # Decode the predicted answer out answer_outputs = LinearRegression(units=n_answer, reg_scale=REG_SCALE, trainable=trainable)(input) model = Model([in_context, in_question], answer_outputs) return model
def commandsmodel(): '''Contains the deep learning model for the commands classification Returns: model -- tensorflow.keras model instance''' model = Sequential() # first layer (Conv1D) model.add( Conv1D(8, kernel_size=13, strides=1, padding='valid', input_shape=(8000, 1))) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.3)) # Second layer(Second Conv1D layer) model.add(Conv1D(16, kernel_size=11, padding='valid', strides=1)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.3)) # third layer(third Conv1D layer) model.add(Conv1D(32, kernel_size=9, padding='valid', strides=1)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.3)) # fourth layer(fourth Conv1D layer) model.add(Conv1D(64, kernel_size=9, padding='valid', strides=1)) model.add(Activation('relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.3)) # fifth layer a Gru layer model.add(GRU(128, return_sequences=True)) model.add(Dropout(0.8)) model.add(BatchNormalization()) # sixth layer (GRU) model.add(GRU(128, return_sequences=True)) model.add(Dropout(0.8)) model.add(BatchNormalization()) # flatten layer model.add(Flatten()) # Dense layer 1 model.add(Dense(256, activation='relu')) model.add(Dropout(0.3)) # Dense layer 2 model.add(Dense(128, activation='relu')) model.add(Dropout(0.3)) # output layer model.add(Dense(9, activation='softmax')) opt = Adam() model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) model.summary() return model
for line in f: values = line.split(); word = values[0]; coefs = np.asarray(values[1:], dtype='float32'); embeddings_index[word] = coefs; embeddings_matrix = np.zeros((vocab_size+1, embedding_dim)); for word, i in word_index.items(): embedding_vector = embeddings_index.get(word); if embedding_vector is not None: embeddings_matrix[i] = embedding_vector; decoder_state=encoder_model.get_layer('dense') decoder_initial_state=decoder_state.output decoder_input=Input(shape=(None,),name='decoder_input') decoder_embedding=Embedding(vocab_size+1, embedding_dim, input_length=decoder_max_len-1, weights=[embeddings_matrix], trainable=False)(decoder_input) decoder_gru1=GRU(256,return_sequences = True)(decoder_embedding,initial_state=decoder_initial_state) decoder_gru3=GRU(256,return_sequences = True)(decoder_gru1,initial_state=decoder_initial_state) decoder_output=Dense(10000,activation='softmax',name='decoder_output')(decoder_gru3) encoder_input=encoder_model.input model_train=Model([encoder_input,decoder_input],[decoder_output]) encoder_input=model_train.get_layer('input_1') encoder_input._name='encoder_input' model_train.summary() model_train.compile(loss='sparse_categorical_crossentropy',optimizer=RMSprop(lr=1e-3),metrics=['accuracy']) path_checkpoint = r'C:\ml\image captioning\31296_39911_bundle_archive\checkpoint_weights' callback_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=path_checkpoint, verbose=1, save_weights_only=True) callback_tensorboard = tf.keras.callbacks.TensorBoard(log_dir='C:\ml\image captioning\31296_39911_bundle_archive\callback_tensor',
from tensorflow.keras import Sequential, Input, Model from tensorflow.keras.layers import Masking, Dense, Activation, GRU, Dropout, concatenate,LSTM from time import time # TF_CONFIG is the environment variable used to configure tf.distribute # each worker will have a different number, an index entry in the nodes_endpoint list # node 0 is master, by default strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # This implements the distributed stratedy for model with strategy.scope(): ## GRU branch gru_input = Input(shape=(801,19), name='gru_input') a = gru_input a = Masking(mask_value=0.)(a) a = GRU(units=50,activation='tanh')(a) gruBranch = Dropout(0.2)(a) hlf_input = Input(shape=(14), name='hlf_input') b = hlf_input hlfBranch = Dropout(0.2)(b) c = concatenate([gruBranch, hlfBranch]) c = Dense(25, activation='relu')(c) output = Dense(3, activation='softmax')(c) model = Model(inputs=[gru_input, hlf_input], outputs=output) ## Compile model optimizer = Adam(learning_rate=0.0005*number_workers) loss = 'categorical_crossentropy'
def train_delta(matrix): import numpy as np import tensorflow as tf import pandas as pd import matplotlib.pyplot as plt from sklearn.preprocessing import MinMaxScaler from tensorflow.keras import Model, Input from tensorflow.keras.layers import (Dense, Dropout, GRU, Flatten, GaussianNoise, concatenate) from tensorflow.keras.models import load_model from tensorflow.keras.callbacks import Callback from tensorflow.keras.callbacks import EarlyStopping from tensorflow.keras.callbacks import ModelCheckpoint from tensorflow.keras.optimizers import Adam import supplemental_functions from supplemental_functions import (sampling_fix, prepareinput, prepareinput_nozero, prepareoutput) tf.keras.backend.clear_session() [ newdim, percent_drilled, start, stop, inc_layer1, inc_layer2, data_layer1, data_layer2, dense_layer, range_max, memory, predictions, drop1, drop2, lr, bs, ensemble_count ] = matrix drop1 = drop1 / 100 drop2 = drop2 / 100 inc_layer2 = inc_layer2 / 1000 lr = lr / 10000 percent_drilled = percent_drilled / 100 df = pd.read_csv('F9ADepth.csv') df_target = df.copy() droplist = [ 'nameWellbore', 'name', 'Pass Name unitless', 'MWD Continuous Inclination dega', 'Measured Depth m', 'MWD Continuous Azimuth dega', "Unnamed: 0", "Unnamed: 0.1" ] for i in droplist: df = df.drop(i, 1) for i in list(df): if df[i].count() < 1000: del df[i] info(f'dropped {i}') start = start stop = stop step = 0.230876 X = np.arange(start, stop, step) X = X.reshape(X.shape[0], 1) X = np.arange(start, stop, step) X = X.reshape(X.shape[0], 1) my_data1 = sampling_fix(df_target, 'MWD Continuous Inclination dega', start, stop, 1.7, 1, 0).predict(X) data_array = [] for i in list(df): sampled = sampling_fix(df_target, i, start, stop, 1.7, 3, 0).predict(X) if np.isnan(np.sum(sampled)) == False: data_array.append(sampled) info(f'Using {i}') data_array = np.asarray(data_array) dftemp = pd.DataFrame() dftemp['dinc'] = my_data1 dftemp['dinc'] = dftemp['dinc'].diff(1).rolling(3, center=True).mean() my_data1 = dftemp['dinc'].ffill().bfill() data_array = data_array.T pre_PCA_scaler = MinMaxScaler() data_array = pre_PCA_scaler.fit_transform(data_array) from sklearn.decomposition import PCA # ============================================================================= # pca = PCA().fit(data_array) # plt.plot(np.cumsum(pca.explained_variance_ratio_)) # plt.xlabel('number of components') # plt.ylabel('cumulative explained variance'); # # plt.show() # ============================================================================= sampcount = int(len(data_array) * percent_drilled) pca = PCA(n_components=newdim).fit(data_array[:sampcount]) projected = pca.transform(data_array) my_data = [] for i in range(newdim): my_data.append(projected[:, i]) my_data1 = my_data1[:, np.newaxis] my_data_newaxis = [] for i in my_data: my_data_newaxis.append(i[:, np.newaxis]) temp_data1 = pd.DataFrame(my_data1.flatten()) temp_data1 = pd.DataFrame(my_data1) range1 = temp_data1[0].diff(memory + predictions) range2 = np.amax(range1) RNN_scaler = MinMaxScaler() my_data1 = RNN_scaler.fit_transform(my_data1) my_data_scaled = [] for i in my_data_newaxis: my_data_scaled.append(MinMaxScaler().fit_transform(i)) X1 = prepareinput(my_data1, memory) Xdata = [] for i in my_data_scaled: Xn = prepareinput_nozero(i, memory, predictions) Xdata.append(Xn) y_temp = prepareoutput(my_data1, memory, predictions) stack = [] for i in range(memory): stack.append(np.roll(my_data1, -i)) X_temp = np.hstack(stack) y = y_temp data_length = len(my_data1) - memory - predictions testing_cutoff = 0.80 border1 = int((data_length) * (percent_drilled * 0.8)) border2 = int((data_length) * (percent_drilled)) border3 = int((data_length) * (percent_drilled + 0.2)) X1_train = X1[:border1] X1_test = X1[border1:border2] X1_test2 = X1[border2:border3] Xdata_train = [] Xdata_test = [] Xdata_test2 = [] for i in Xdata: Xdata_train.append(i[:border1]) Xdata_test.append(i[border1:border2]) Xdata_test2.append(i[border2:border3]) y_train, y_test, y_test2 = y[:border1], y[border1:border2], y[ border2:border3] X1_train = X1_train.reshape((X1_train.shape[0], X1_train.shape[1], 1)) X1_test = X1_test.reshape((X1_test.shape[0], X1_test.shape[1], 1)) X1_test2 = X1_test2.reshape((X1_test2.shape[0], X1_test2.shape[1], 1)) Xdata_train_r = [] Xdata_test_r = [] Xdata_test2_r = [] for i in range(newdim): Xdata_train_r.append(Xdata_train[i].reshape( (Xdata_train[i].shape[0], Xdata_train[i].shape[1], 1))) Xdata_test_r.append(Xdata_test[i].reshape( (Xdata_test[i].shape[0], Xdata_test[i].shape[1], 1))) Xdata_test2_r.append(Xdata_test2[i].reshape( (Xdata_test2[i].shape[0], Xdata_test2[i].shape[1], 1))) X_train_con = np.concatenate(Xdata_train_r, axis=2) X_test_con = np.concatenate(Xdata_test_r, axis=2) X_test2_con = np.concatenate(Xdata_test2_r, axis=2) X_train = [X1_train, X_train_con] X_test = [X1_test, X_test_con] X_test2 = [X1_test2, X_test2_con] input1 = Input(shape=(memory, 1)) input2 = Input(shape=(memory + predictions, newdim)) x1 = GaussianNoise(inc_layer2, input_shape=(memory, 1))(input1) x1 = GRU(units=inc_layer1, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer='l2', recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, stateful=False)(x1) x1 = Dropout(drop1)(x1) x1 = Model(inputs=input1, outputs=x1) x2 = Dense(data_layer1, input_shape=(memory + predictions, 3))(input2) x2 = Dropout(drop2)(x2) x2 = Flatten()(x2) x2 = Dense(data_layer2)(x2) x2 = Model(inputs=input2, outputs=x2) combined = concatenate([x1.output, x2.output]) z = Dense(dense_layer, activation="relu")(combined) z = Dense(predictions, activation="linear")(z) #define the model myadam = Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, amsgrad=False) class PlotResuls(Callback): def on_train_begin(self, logs={}): self.i = 0 self.x = [] self.losses = [] self.val_losses = [] #self.fig = plt.figure() self.logs = [] def on_epoch_end(self, epoch, logs={}): self.logs.append(logs) self.x.append(self.i) self.losses.append(logs.get('loss')) self.val_losses.append(logs.get('val_loss')) self.i += 1 #print (".", end = '') if (epoch % 14999 == 0) & (epoch > 0): print(epoch) plt.plot(self.x, np.log(self.losses), label="loss") plt.plot(self.x, np.log(self.val_losses), label="val_loss") plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2) plt.title("Loss") plt.legend() plt.show() #mymanyplots(epoch, data, model) #data = [X1, X2, X3, X4, y, X1_train,X_train, X_test, X1_test, border1, border2, y_train, y_test, memory, y_temp, predictions] plot_results = PlotResuls() es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=25) ens_val_array = np.zeros(ensemble_count) ens_test_array = np.zeros(ensemble_count) for ens_no in range(ensemble_count): tf.keras.backend.clear_session() mc = ModelCheckpoint(f'best_model_ens_{ens_no}.h5', monitor='val_loss', mode='min', save_best_only=True, verbose=0) model = Model(inputs=[x1.input, x2.input], outputs=z) model.compile(optimizer=myadam, loss='mean_squared_error') history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2000, verbose=0, batch_size=bs, callbacks=[plot_results, es, mc]) model = load_model(f'best_model_ens_{ens_no}.h5') valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0)) testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0)) ens_val_array[ens_no] = valresult ens_test_array[ens_no] = testresult winner = ens_val_array.argmin() model = load_model(f'best_model_ens_{winner}.h5') info(ens_val_array) info(ens_test_array) info(f'Validation winner {winner}') sample_count = len(X_test2[0]) y_pred = model.predict(X_test2) plt.plot(np.log(history.history['loss']), label='loss') plt.plot(np.log(history.history['val_loss']), label='test') plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2) plt.legend() plt.clf() plt.show() for i in range(5): rand = np.random.randint(0, len(X_test[0])) y_test_descaled = RNN_scaler.inverse_transform(y_test[rand, np.newaxis]) y_in_descaled = RNN_scaler.inverse_transform(X_test[0][rand, :]) y_in_descaled = y_in_descaled.flatten() y_test_descaled = y_test_descaled.flatten() y_pred = model.predict(X_test) y_pred_descaled = RNN_scaler.inverse_transform(y_pred[rand, np.newaxis]) y_pred_descaled = y_pred_descaled.flatten() plt.plot(y_test_descaled, label="true") plt.plot(y_pred_descaled, label="predicted") plt.title('Inclination delta') #plt.ylim(0,1) plt.legend() plt.show() plt.figure(figsize=(5, 4)) x_after = np.linspace(0, 23, 100) x_before = np.linspace(-23, -0.23, 100) plt.plot(x_before, np.cumsum(y_in_descaled), label="measured", linestyle="-", c="black") commonpoint = np.cumsum(y_in_descaled)[-1] plt.plot(x_after, commonpoint + np.cumsum(y_test_descaled), label="actual", linestyle='-.', c='black') plt.plot(x_after, commonpoint + np.cumsum(y_pred_descaled), label="predicted", linestyle=':', c='black') #plt.title('') plt.ylim(-1, 7) plt.grid() plt.tight_layout() #plt.hlines(0, -23, 23, linewidth=0.5) plt.xlabel("Distance to sensor [m]") plt.ylabel("Inclination, local coordinates, [deg]") plt.legend() plt.tight_layout() plt.savefig(f'Sample, {percent_drilled}, no.{i}.pdf') plt.show() # #### Different ensemble, voting ###### # ypred_array = [] # for i in range(ensemble_count): # model = load_model(f'best_model_ens_{i}.h5') # y_pred = model.predict(X_test2) # ypred_array.append(y_pred) # y_pred = np.average(ypred_array, axis=0) # ######## Different ensemble ends here # y_test_descaled = RNN_scaler.inverse_transform(y_test2) y_pred = model.predict(X_test2) y_pred_descaled = RNN_scaler.inverse_transform(y_pred) error_matrix = np.cumsum(y_pred_descaled, axis=1) - np.cumsum( y_test_descaled, axis=1) def rand_jitter(arr): stdev = .004 * (max(arr) - min(arr)) return arr + np.random.randn(len(arr)) * stdev def jitter(x, y, s=20, c='b', marker='o', cmap=None, norm=None, vmin=None, vmax=None, alpha=None, linewidths=None, verts=None, **kwargs): return plt.scatter(rand_jitter(x), rand_jitter(y), s=s, c=c, marker=marker, cmap=cmap, norm=norm, vmin=vmin, vmax=vmax, alpha=alpha, linewidths=linewidths, verts=verts, **kwargs) # ============================================================================= # plt.figure(figsize=(5,5), dpi=200) # for i in range(sample_count): # _ = jitter(x_after,error_matrix[i], alpha=1,s=0.5,marker=".", c="black") # plt.title(f"delta, drilled {percent_drilled}") # plt.xlabel("Distance to sensor [m]") # plt.ylabel("Prediction error [deg]") # plt.grid() # plt.tight_layout() # plt.savefig(f'Birdflock, {percent_drilled}.pdf') # plt.show() # #plt.plot(np.median(error_matrix, axis=0), linewidth=8, alpha=1, c="white") # #plt.plot(np.median(error_matrix, axis=0), linewidth=2, alpha=1, c="black") # plt.scatter(np.arange(0,100,1),np.average(np.abs(error_matrix), axis=0), # marker="o",s=40, alpha=0.7, c="white", zorder=2) # # ============================================================================= c_array = np.empty(100, dtype=object) aae = np.average(np.abs(error_matrix), axis=0) # ============================================================================= # for i in range(100): # if aae[i] <= 0.4: # c_array[i] = "green" # elif aae[i] <= 0.8: # c_array[i] = "orange" # else: # c_array[i] = "red" # # plt.scatter(np.arange(0,100,1),aae, marker=".",s=20, alpha=1, c=c_array, # zorder=3, label="Average Absolute Error") # plt.ylim((-3,3)) # plt.axhline(y=0, xmin=0, xmax=1, linewidth=2, c="black") # plt.axhline(y=0.4, xmin=0, xmax=1, linewidth=1, c="black") # plt.axhline(y=0.8, xmin=0, xmax=1, linewidth=1, c="black") # plt.legend() # plt.show() # ============================================================================= model = load_model('best_model.h5') #mymanyplots(-1, data, model) #myerrorplots(data, model) valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0)) testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0)) return valresult, testresult, aae
test_x, test_y = test[:, :-1], test[:, -1] # 为了在LSTM中应用该数据,需要将其格式转化为3D format,即[Samples, timesteps, features] train_X = train_x.reshape((train_x.shape[0], 1, train_x.shape[1])) test_X = test_x.reshape((test_x.shape[0], 1, test_x.shape[1])) model = Sequential() model.add( Conv1D(filters=32, kernel_size=3, strides=1, padding="causal", activation="relu")) model.add( GRU(32, input_shape=(train_X.shape[1], train_X.shape[2]), return_sequences=True)) model.add(GRU(16, input_shape=(train_X.shape[1], train_X.shape[2]))) model.add(Dense(16, activation="relu")) model.add(Dropout(0.2)) model.add(Dense(1)) model.compile(loss=tf.keras.losses.Huber(), optimizer='adam', metrics=["mse"]) history = model.fit(train_X, train_y, epochs=50, batch_size=72, validation_data=(test_X, test_y), verbose=2) # 画图 plt.plot(history.history['loss'], label='train')
trainset = dataset_scaled[:len(dataset_train)] testset = dataset_scaled[-len(dataset_test)-60:] x_train = [] y_train = [] for i in range(60,1259): x_train.append(trainset[i-60:i, 0]) y_train.append(trainset[i,0]) x_train,y_train = np.array(x_train),np.array(y_train) x_train = np.reshape(x_train, (x_train.shape[0],x_train.shape[1],1)) # building model regressor = Sequential() regressor.add(GRU(units = 50,return_sequences = True,input_shape = (x_train.shape[1],1))) regressor.add(Dropout(0.2)) regressor.add(GRU(units = 50,return_sequences = True)) regressor.add(Dropout(0.2)) regressor.add(GRU(units = 50)) regressor.add(Dropout(0.2)) regressor.add(Dense(units = 1)) regressor.compile(optimizer = 'adam',loss = 'mean_squared_error') regressor.fit(x_train,y_train,epochs = 100, batch_size = 32) # predict in test set x_test = [] for i in range(60,185): x_test.append(testset[i-60:i,0])
def build_configurable_model(self): if self.cell_type == "RNN": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = SimpleRNN( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state = encoder(encoder_inputs) encoder_states = [state] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = SimpleRNN( self.latentDim, return_sequences=True, return_state=True, dropout=self.dropout, ) decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states) # dense hidden = Dense(self.hidden, activation="relu") hidden_outputs = hidden(decoder_outputs) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model elif self.cell_type == "LSTM": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = LSTM( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state_h, state_c = encoder(encoder_outputs) encoder_states = [state_h, state_c] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = LSTM( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) decoder_outputs, _, _ = decoder( decoder_outputs, initial_state=encoder_states ) # dense hidden = Dense(self.hidden, activation="relu") hidden_outputs = hidden(decoder_outputs) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model elif self.cell_type == "GRU": # encoder encoder_inputs = Input(shape=(None, len(self.srcChar2Int))) encoder_outputs = encoder_inputs for i in range(1, self.numEncoders + 1): encoder = GRU( self.latentDim, return_state=True, return_sequences=True, dropout=self.dropout, ) encoder_outputs, state = encoder(encoder_inputs) encoder_states = [state] # decoder decoder_inputs = Input(shape=(None, len(self.tgtChar2Int))) decoder_outputs = decoder_inputs for i in range(1, self.numDecoders + 1): decoder = GRU( self.latentDim, return_sequences=True, return_state=True, dropout=self.dropout, ) decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states) # dense hidden = Dense(self.hidden, activation="relu") hidden_outputs = hidden(decoder_outputs) decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax") decoder_outputs = decoder_dense(hidden_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) return model
print(y_target.shape) # (1065, 2, 1) from sklearn.model_selection import train_test_split x1_train, x1_test, x2_train, x2_test, y_train, y_test = train_test_split(x1_data[:-1], x2_data[:-1], y_target, test_size=0.2, random_state=45) x1_train, x1_val, x2_train, x2_val, y_train, y_val = train_test_split(x1_train, x2_train, y_train, test_size=0.2, random_state=45) print(x1_train.shape, x1_test.shape, x1_val.shape) print(x2_train.shape, x2_test.shape, x2_val.shape) print(y_train.shape, y_test.shape, y_val.shape) # 모델 from tensorflow.keras.models import Model from tensorflow.keras.layers import Input, Dense, LSTM, GRU, Conv1D, MaxPooling1D, Flatten, Dropout, Concatenate input1 = Input(shape=(x1_data.shape[1], x1_data.shape[2])) layer1 = GRU(64)(input1) layer1 = Dropout(0.2)(layer1) layer1 = Dense(32)(layer1) input2 = Input(shape=(x2_data.shape[1], x2_data.shape[2])) layer2 = Conv1D(16, 3, padding='same', strides=1, activation='relu')(input2) layer2 = MaxPooling1D(2)(layer2) layer2 = Dropout(0.2)(layer2) layer2 = Flatten()(layer2) layer2 = Dense(16)(layer2) merge = Concatenate()([layer1, layer2]) merge = Dense(64, activation='relu')(merge) merge = Dense(64, activation='relu')(merge) output1 = Dense(2, activation='relu')(merge)
#读取数据集 train_x, test_x, train_y, test_y = util.load_data2('dataset_all_420.csv') # 训练集 print(train_x.shape) print(train_y.shape) print(test_x.shape) epochs = 1000 batch_size = 5 # GRU参数: return_sequences=True GRU输出为一个序列。默认为False,输出一个值。 # input_dim: 输入单个样本特征值的维度 # input_length: 输入的时间点长度 model = Sequential() model.add( GRU(units=10, return_sequences=True, input_dim=train_x.shape[-1], input_length=train_x.shape[1])) model.add(GRU(units=50)) model.add(Dense(1)) model.compile(loss='mean_squared_error', optimizer='adam') model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=1) y_pred = model.predict(test_x) rms = np.sqrt(np.mean(np.power((test_y - y_pred), 2))) print(rms) print(y_pred.shape) print(test_y.shape) x_axis = np.arange(1, np.shape(test_y)[0] + 1) plt.xlabel('Time') plt.ylabel('Price')
x_input = np.array([50, 60, 70]) # (3, ) print(x.shape) print(y.shape) print(x_input.shape) x = x.reshape(13, 3, 1) print(x.shape) # y = y.reshape(13) # print(y.shape) # 2. 모델구성 from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, LSTM, SimpleRNN, GRU model = Sequential() model.add(GRU(200, activation='relu', input_shape=(3, 1))) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(1)) model.summary() # 3. 컴파일, 훈련 model.compile(loss='mse', optimizer='adam', metrics='mae') model.fit(x, y, epochs=200, batch_size=1)
def main(job_dir, **args): gcp = args['gcp'] latent_unit_count = 512 EPOCHS = 500 BATCH_SIZE = 96 dropout_rate = 0.3 opt = 'adam' output_dir = f"final" history_name = f'training_history.json' if not os.path.exists(output_dir): os.makedirs(output_dir) path = lambda x: os.path.join(output_dir, x) data_x, data_y, dataset_encoder = load_data(job_dir=job_dir, gcp=gcp, shuffle_data=True, augment=True, mode="crop") with open('encoder.pickle', 'wb') as f: pickle.dump(dataset_encoder, f, pickle.HIGHEST_PROTOCOL) sys.exit(0) total_n_songs = data_x.shape[0] validation_split = 0.2 # test_split = 0.1 # test_split_idx = int(total_n_songs * (1 - test_split)) # test_x, test_y = data_x[test_split_idx:], data_y[test_split_idx:] # data_x, data_y = data_x[:test_split_idx], data_y[:test_split_idx] validation_idx = int(total_n_songs * (1 - validation_split)) train_x, train_y = data_x[:validation_idx], data_y[:validation_idx] val_data = data_x[validation_idx:], data_y[validation_idx:] train_gen = make_generator(data_x, data_y, BATCH_SIZE) val_gen = make_generator(val_data[0], val_data[1], BATCH_SIZE) n_songs, song_len, n_notes = train_x.shape inputs = Input(shape=(song_len, n_notes), name="melody_input") input_encoder = GRU( units=latent_unit_count, return_sequences=True, return_state=True, name="melody_encoder", dropout=dropout_rate, ) x, input_state = input_encoder(inputs) alto_encoder = GRU(units=latent_unit_count, return_sequences=True, return_state=True, name="alto_encoder", dropout=dropout_rate) tenor_encoder = GRU(units=latent_unit_count, return_sequences=True, return_state=True, name="tenor_encoder", dropout=dropout_rate) bass_encoder = GRU(units=latent_unit_count, return_sequences=True, return_state=True, name="bass_encoder", dropout=dropout_rate) a, a_state = alto_encoder(x, initial_state=input_state) t, t_state = tenor_encoder(x, initial_state=input_state) b, b_state = bass_encoder(x, initial_state=input_state) alto_decoder = GRU(units=latent_unit_count, return_sequences=True, name="alto_decoder", dropout=dropout_rate) tenor_decoder = GRU( units=latent_unit_count, return_sequences=True, name="tenor_decoder", dropout=dropout_rate, ) bass_decoder = GRU( units=latent_unit_count, return_sequences=True, name="bass_decoder", dropout=dropout_rate, ) a = alto_decoder(a, initial_state=a_state) t = tenor_decoder(t, initial_state=t_state) b = bass_decoder(b, initial_state=b_state) reshaper = Reshape((song_len, 1, latent_unit_count)) a = reshaper(a) t = reshaper(t) b = reshaper(b) x = Concatenate(axis=2)([a, t, b]) merger = Dense(n_notes, name="fc_output") x = merger(x) outputs = Activation("softmax", name="softmax")(x) model = Model(inputs=inputs, outputs=outputs, name="BachNet") model.compile(loss="categorical_crossentropy", optimizer=opt) model.summary() try: tf.keras.utils.plot_model( model, to_file=path("model.png"), dpi=1200, show_shapes=True, show_layer_names=True, rankdir="TB", ) except ImportError: print('Graphviz/pydot not installed, skipping model plot...') sys.exit(0) callbacks = [ tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True, min_delta=0, monitor='val_loss'), # tf.keras.callbacks.ModelCheckpoint( # filepath=path("model.{epoch:02d}-{val_loss:.2f}.h5"), # monitor="val_loss", # save_best_only=True, # ), # tf.keras.callbacks.TensorBoard(log_dir="./logs"), ] hist = model.fit( train_gen, validation_data=val_data, # validation_split=validation_split, steps_per_epoch=n_songs // BATCH_SIZE, # batch_size=BATCH_SIZE, epochs=EPOCHS, callbacks=callbacks, # shuffle=True, ) if gcp: # save the model on to google cloud storage save_path = path('final_model.h5') model.save(save_path) # model.save_weights(path) with file_io.FileIO(save_path, mode='rb') as in_f: with file_io.FileIO(os.path.join(job_dir, save_path), mode='wb+') as of: of.write(in_f.read()) else: import matplotlib.pyplot as plt model.save(path("final_model.h5")) plt.plot(hist.history["loss"], label="Training loss") plt.plot(hist.history["val_loss"], label="Validation loss") plt.xlabel('Epoch') plt.ylabel('Cross-entropy loss') plt.legend() plt.savefig(path("history.png"), dpi=300) with open(path(history_name), "w") as f: json.dump( { "loss": hist.history["loss"], }, f #"val_loss": hist.history["val_loss"]}, f ) plt.show()
'helper function to create x, c and y with proper shapes' x = data.filter(like='x-', axis=1).values[:, :, np.newaxis] c = data[top_cities[1:]].to_numpy() y = data.Amsterdam.values[:, np.newaxis] return x, c, y # create correct shapes for tensorflow x_train, c_train, y_train = create_xy(train) x_test, c_test, y_test = create_xy(test) # deterministic set_seed(random_state) # As before, I start out by a pure autoregressive model. model = Sequential(layers=[GRU(cells), Dense(units=1, activation='linear')]) model.compile(optimizer='adam', loss='mae') history = model.fit(x=x_train, y=y_train, epochs=epochs, batch_size=None, shuffle=True, validation_split=validation_split) # The final test loss is; def inverseAms(data): return (ct.named_transformers_['Amsterdam'] .inverse_transform(data) ) modelmae = mean_absolute_error(inverseAms(model.predict(x_test)),
def _crnn_layers(self, input_shape, output_shape): channel_axis = 3 melgram_input = Input(shape=input_shape, dtype='float32') # Input block padding = self.network_input_width - input_shape[1] left_pad = int(padding / 2) if padding % 2: right_pad = left_pad + 1 else: right_pad = left_pad input_padding = ((0, 0), (left_pad, right_pad)) hidden = ZeroPadding2D(padding=input_padding)(melgram_input) # Conv block 1 hidden = Conv2D(64, (3, 3), padding=self.padding, name='conv1')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn1')(hidden) hidden = Activation('elu', name='elu-1')(hidden) hidden = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(hidden) hidden = Dropout(0.1, name='dropout1')(hidden) # Conv block 2 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv2')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn2')(hidden) hidden = Activation('elu', name='elu-2')(hidden) hidden = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(hidden) hidden = Dropout(0.1, name='dropout2')(hidden) # Conv block 3 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv3')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn3')(hidden) hidden = Activation('elu', name='elu-3')(hidden) hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(hidden) hidden = Dropout(0.1, name='dropout3')(hidden) # Conv block 4 hidden = Conv2D(128, (3, 3), padding=self.padding, name='conv4')(hidden) hidden = BatchNormalization(axis=channel_axis, name='bn4')(hidden) hidden = Activation('elu', name='elu-4')(hidden) hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(hidden) hidden = Dropout(0.1, name='dropout4')(hidden) # reshaping hidden = Reshape((15, 128))(hidden) # GRU block 1, 2, output embed_size = 32 hidden = GRU(embed_size, return_sequences=True, name='gru1')(hidden) hidden = GRU(embed_size, return_sequences=self.attention, name='gru2')(hidden) if self.attention: attention = Dense(1)(hidden) attention = Flatten()(attention) attention_act = Activation('softmax')(attention) attention = RepeatVector(embed_size)(attention_act) attention = Permute((2, 1))(attention) merged = Multiply()([hidden, attention]) hidden = Lambda(lambda xin: K.sum(xin, axis=1))(merged) if self.output_dropout: hidden = Dropout(self.output_dropout)(hidden) output = Dense(output_shape, activation='sigmoid', name='crnn_output')(hidden) return melgram_input, output
y = [] print(len(data)) print("The last sample would be: " + str(len(data)) + " - " + str(T) + " = " + str((len(data) - T))) for i in range(len(data) - T): x = data[i:i + T] X.append(x) y_temp = data[i + T] y.append(y_temp) X = np.array(X).reshape(-1, T, D) y = np.array(y) i_layer = Input(shape=(T, D)) h_layer = GRU(10, activation='tanh')(i_layer) o_layer = Dense(1)(h_layer) model = Model(i_layer, o_layer) model.compile(loss='mse', optimizer=Adam(lr=0.2)) index = -N // 4 report = model.fit(X[:index], y[:index], epochs=50, validation_data=(X[index:], y[index:])) plt.plot(report.history['loss'], label='training_loss') plt.plot(report.history['val_loss'], label='validation_loss') plt.legend() y_test = y[index:]