def define_architecture(input_shape, n_classes): model = Sequential() model.add(BatchNormalization(input_shape=input_shape)) model.add(Conv2D(30, (3,3), activation='elu', padding='same')) model.add(MaxPooling2D((2, 2))) model.add(Dropout(0.1)) model.add(BatchNormalization()) model.add(Conv2D(60, (3,3), activation='elu', padding='same')) model.add(MaxPooling2D((2, 3))) model.add(Dropout(0.1)) model.add(BatchNormalization()) model.add(Conv2D(60, (3,3), activation='elu', padding='same')) model.add(MaxPooling2D((4, 4))) model.add(Dropout(0.1)) model.add(BatchNormalization()) model.add(Conv2D(60, (3,3), activation='elu', padding='same')) model.add(MaxPooling2D((4, 4))) model.add(Dropout(0.1)) model.add(TimeDistributed(SimpleRNN(30))) model.add(Flatten()) model.add(Dense(n_classes, activation='softmax')) return model
def RNN(x_train, x_eval, y_train, y_eval, x_test):
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, SimpleRNN
model = Sequential()
model.add(SimpleRNN(32, activation='relu', input_shape=(x_train.shape[1], 1)))
nodes = [32, 16, 8, 1]
for i in nodes:
model.add(Dense(i, activation='relu'))
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
es = EarlyStopping(monitor='val_loss', patience=6, mode='auto')
rlr = ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.3, verbose=1)
pred_return = pd.DataFrame() # [3888 rows x 9 columns], <class 'pandas.core.frame.DataFrame'>
for i in range(len(quantiles)):
from tensorflow.keras.optimizers import Adam
model.compile(loss = lambda y_true, y_pred: quantile_loss_dacon(quantiles[i], y_true, y_pred), optimizer=Adam(learning_rate=0.001), metrics=['mae'])
model.fit(x_train, y_train, epochs=100, batch_size=32, callbacks=[es, rlr], validation_data=(x_eval, y_eval))
pred = model.predict(x_test) # <class 'numpy.ndarray'>, (3888, 1)
pred_rs = pred.reshape(pred.shape[0]) # (3888, )
pred_pd = pd.Series(pred_rs)
pred_return = pd.concat([pred_return, pred_pd], axis=1)
return pred_return, model
def CNN_model(recurrent_model=None, filter_num=3):
if recurrent_model:
model_input = Input(batch_shape=(1, None, 1))
else:
model_input = Input(shape=(None, 1))
submodels = []
for kw in range(3, 3 + filter_num):
conv = Conv1D(100,
kernel_size=(kw * embed_size, ),
padding='valid',
activation='relu',
kernel_regularizer=l2(0.01),
strides=embed_size)(model_input)
conv = GlobalMaxPooling1D()(conv)
submodels.append(conv)
z = Concatenate()(submodels)
z = Dropout(0.3)(z)
if recurrent_model:
z = Reshape(target_shape=((1, 300)))(z)
if recurrent_model == 'gru':
z = GRU(128,
stateful=True,
batch_input_shape=(1, 300, 1),
dropout=0.2,
recurrent_dropout=0.2)(z)
elif recurrent_model == 'lstm':
z = LSTM(128, stateful=False)(z)
elif recurrent_model == 'rnn':
z = SimpleRNN(128, stateful=False)(z)
z = Dense(100, activation='relu', kernel_regularizer=l2(0.01))(z)
z = Dropout(0.3)(z)
model_output = Dense(1, activation='sigmoid')(z)
model = Model(model_input, model_output)
#model.summary()
model.compile(optimizer='adam', loss=KL_dv_loss, metrics=['acc'])
return model
def createModel(self):
self.model_instance += 1
clear_session()
features, label = self.getDataset()
X_train, y_train = self.createLag(features, label)
X_train = X_train[:, self.lags]
learning_rate = float(self.hyperparameters["Learning_Rate"].get())
momentum = float(self.hyperparameters["Momentum"].get())
optimizers = {
"Adam": Adam(learning_rate=learning_rate),
"SGD": SGD(learning_rate=learning_rate, momentum=momentum),
"RMSprop": RMSprop(learning_rate=learning_rate, momentum=momentum)
}
shape = (X_train.shape[1], X_train.shape[2])
model_choice = self.model_var.get()
if not self.do_optimization:
model = Sequential()
model.add(Input(shape=shape))
if model_choice == 0:
model.add(Flatten())
layers = self.no_optimization_choice_var.get()
for i in range(layers):
neuron_number = self.neuron_numbers_var[i].get()
activation_function = self.activation_var[i].get()
if model_choice == 0:
model.add(Dense(neuron_number, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
elif model_choice == 1:
model.add(Conv1D(filters=neuron_number, kernel_size=2, activation=activation_function, kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
elif model_choice == 2:
if i == layers-1:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 3:
if i == layers-1:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
else:
model.add(Bidirectional(LSTM(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0))))
model.add(Dropout(0.2))
elif model_choice == 4:
if i == layers-1:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(SimpleRNN(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
elif model_choice == 5:
if i == layers-1:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=False, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
else:
model.add(GRU(neuron_number, activation=activation_function, return_sequences=True, kernel_initializer=GlorotUniform(seed=0), recurrent_initializer=Orthogonal(seed=0)))
model.add(Dropout(0.2))
if model_choice == 1:
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, activation=self.output_activation.get(), kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1, shuffle=False)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
elif self.do_optimization:
layer = self.optimization_choice_var.get()
if model_choice == 0:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
model.add(Flatten())
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Dense(units=hp.Int('MLP_'+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu'))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". MLP"
elif model_choice == 1:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max-n_min)/4)
model.add(Conv1D(filters=hp.Int("CNN_"+str(i), min_value=n_min, max_value=n_max, step=step), kernel_size=2, activation="relu", kernel_initializer=GlorotUniform(seed=0)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(32, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". CNN"
elif model_choice == 2:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0)))
if i == layer-1:
model.add(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0)))
model.add(Dense(1))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". LSTM"
elif model_choice == 3:
def build_model(hp):
model = Sequential()
model.add(Input(shape=shape))
for i in range(layer):
n_min = self.neuron_min_number_var[i].get()
n_max = self.neuron_max_number_var[i].get()
step = int((n_max - n_min)/4)
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=True, kernel_initializer=GlorotUniform(seed=0))))
if i == layer-1:
model.add(Bidirectional(LSTM(units=hp.Int("LSTM_"+str(i), min_value=n_min, max_value=n_max, step=step), activation='relu', return_sequences=False, kernel_initializer=GlorotUniform(seed=0))))
model.add(Dense(1, kernel_initializer=GlorotUniform(seed=0)))
model.compile(optimizer = optimizers[self.hyperparameters["Optimizer"].get()], loss=self.hyperparameters["Loss_Function"].get())
return model
name = str(self.model_instance) + ". Bi-LSTM"
tuner = RandomSearch(build_model, objective='loss', max_trials=25, executions_per_trial=2, directory=self.runtime, project_name=name)
tuner.search(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get())
hps = tuner.get_best_hyperparameters(num_trials = 1)[0]
model = tuner.hypermodel.build(hps)
history = model.fit(X_train, y_train, epochs=self.hyperparameters["Epoch"].get(), batch_size=self.hyperparameters["Batch_Size"].get(), verbose=1)
loss = history.history["loss"][-1]
self.train_loss.set(loss)
for i in range(layer):
if model_choice == 0:
self.best_model_neurons[i].set(model.get_layer(index=i+1).get_config()["units"])
elif model_choice == 1:
self.best_model_neurons[i].set(model.get_layer(index=(2*i)).get_config()["filters"])
elif model_choice == 2:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["units"])
elif model_choice == 3:
self.best_model_neurons[i].set(model.get_layer(index=i).get_config()["layer"]["config"]["units"])
model.summary()
self.model = model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, SimpleRNN
#실습: LSTM 완성
#예측값: 80
model = Sequential()
# model.add(LSTM(200, input_shape=(3, 1)))
# model.add(Dense(100))
# model.add(Dense(50))
# model.add(Dense(30))
# model.add(Dense(10))
# model.add(Dense(1))
#과제: 80.00809
model.add(SimpleRNN(30, activation='relu', input_shape=(3, 1)))
model.add(Dense(70))
model.add(Dense(100))
model.add(Dense(50))
model.add(Dense(30))
model.add(Dense(10))
model.add(Dense(1))
#3. 컴파일, 훈련
model.compile(loss='mse', optimizer='adam', metrics='mse')
model.fit(x, y, epochs=200, batch_size=1)
#4. 평가, 예측
loss, acc = model.evaluate(x, y, batch_size=1)
print("\nloss: ", loss)
x_train, y_train = np.array(x_train), np.array(y_train)
# 使x_train符合RNN输入要求：[送入样本数， 循环核时间展开步数， 每个时间步输入特征个数]。
# 此处整个数据集送入，送入样本数为x_train.shape[0]即2066组数据；输入60个开盘价，预测出第61天的开盘价(标签)，循环核时间展开步数为60; 每个时间步送入的特征是某一天的开盘价，只有1个数据，故每个时间步输入特征个数为1
x_train = np.reshape(x_train, (x_train.shape[0], 60, 1))
# 测试集：csv表格中后300天数据
# 利用for循环，遍历整个测试集，提取测试集中连续60天的开盘价作为输入特征x_train，第61天的数据作为标签，for循环共构建300-60=240组数据。
for i in range(60, len(test_set)):
x_test.append(test_set[i - 60:i, 0])
y_test.append(test_set[i, 0])
# 测试集变array并reshape为符合RNN输入要求：[送入样本数， 循环核时间展开步数， 每个时间步输入特征个数]
x_test, y_test = np.array(x_test), np.array(y_test)
x_test = np.reshape(x_test, (x_test.shape[0], 60, 1))
model = tf.keras.Sequential([
SimpleRNN(80, return_sequences=True),
Dropout(0.2),
SimpleRNN(100),
Dropout(0.2),
Dense(1)
])
model.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss='mean_squared_error') # 损失函数用均方误差
# 该应用只观测loss数值，不观测准确率，所以删去metrics选项，一会在每个epoch迭代显示时只显示loss值
checkpoint_save_path = "./checkpoint/rnn_stock.ckpt"
if os.path.exists(checkpoint_save_path + '.index'):
print('-------------load the model-----------------')
model.load_weights(checkpoint_save_path)
(input_train, y_train), (input_test, y_test) = imdb.load_data(
num_words=max_features
)
print(len(input_train), 'train sequences')
print(len(input_test), 'test sequences')
print('Pad sequences (samples x time)')
input_train = sequence.pad_sequences(input_train, maxlen=maxlen)
input_test = sequence.pad_sequences(input_test, maxlen=maxlen)
print('input_train shape:', input_train.shape)
print('input_test shape:', input_test.shape)
model = Sequential()
model.add(Embedding(max_features, 32))
model.add(SimpleRNN(32))
model.add(Dense(1, activation='sigmoid'))
model.compile(
optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy']
)
history = model.fit(
input_train,
y_train,
epochs=10,
validation_split=0.2
)
accuracy = history.history['accuracy']
import matplotlib.pyplot as plt
#RNN model
# N = number of samples
# T = sequence length
# D = number of input features
# M = number of hidden units
# K = number of output units
N = 1
T = 1
D = 67
M = 10
K = 2
i = Input(shape=(T, D))
x = SimpleRNN(M, return_sequences=True)(i)
x = GlobalMaxPool1D()(x)
x = Dropout(0.5)(x)
x = Dense(K, activation='relu')(x)
model = Model(i, x)
#model.compile( loss = 'mse', metrics = ['accuracy'], optimizer = Adam(lr = 0.001),)
model.compile(
loss='mse',
optimizer=Adam(lr=0.001),
metrics=['accuracy'],
)
# split the dataset array to X and y
def split_dataset_array(dataset_array, time_step):
X, y = list(), list()
from tensorflow.keras.layers import SimpleRNN, Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.losses import mean_absolute_percentage_error
model = Sequential([
SimpleRNN(units=16, input_shape=(max_len, number_of_features)),
Dense(units=1)
])
model.compile(optimizer='adam', loss=mean_absolute_percentage_error)
state_t = output_t final_output_sequence = np.stack(successive_outputs, axis=0) # (100, 64) final_output_sequence.shape #%% from tensorflow import keras keras.__version__ #%% from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Embedding, SimpleRNN, Dense model = Sequential() model.add(Embedding(10000, 32)) model.add(SimpleRNN(32)) model.summary() model = Sequential() model.add(SimpleRNN(2, return_sequences=True, input_shape=(3, 2))) model.add(SimpleRNN(3)) model.summary() #%% model = Sequential() model.add(Embedding(10000, 32)) model.add(SimpleRNN(32, return_sequences=True)) model.summary() #%%
N = number of samples T = sequence length D = number of input features M = number of hidden units K = number of output units """ N = 1 T = 10 D = 3 K = 2 X = np.random.randn(N, T, D) #%% Make an RNN M = 5 # number of hidden units i = Input(shape=(T, D)) x = SimpleRNN(units=M)(i) # in RNNs default activation is not None it is tanh x = Dense(units=K)(x) model = Model(i,x) #%% Get the output Yhat = model.predict(X) print(Yhat) #%% See if we can replicate this output # Get the weights first model.summary() # See what's returned print(model.layers[1].get_weights()) #%% Check their shapes
x_test, y_test = np.array(x_test), np.array(y_test)
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
print('x_train:', x_train)
print('x_train:', x_train.shape)
model = tf.keras.Sequential([
# layers.Embedding(5, 1,input_length=1),
# SimpleRNN(50, return_sequences=True, unroll=True),
# Dropout(0.2),
# SimpleRNN(50, return_sequences=True, unroll=True),
# Dropout(0.2),
# SimpleRNN(50, return_sequences=True, unroll=False),
# Dropout(0.2),
SimpleRNN(128, unroll=True),
Dropout(0.2),
Dense(1)
])
model.compile(optimizer='adam', loss='mean_squared_error')
checkpoint_save_path = "./checkpoint/stock.ckpt"
if os.path.exists(checkpoint_save_path + '.index'):
print('-------------load the model-----------------')
model.load_weights(checkpoint_save_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_save_path,
save_weights_only=True,
monitor='loss',
from tensorflow.keras.layers import SimpleRNN
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, SimpleRNN
model = Sequential()
model.add(Embedding(10000, 32))
model.add(SimpleRNN(32))
model.summary()
model = Sequential()
model.add(Embedding(10000, 32))
model.add(SimpleRNN(32, return_sequences=True))
model.add(SimpleRNN(32, return_sequences=True))
model.add(SimpleRNN(32, return_sequences=True))
model.add(SimpleRNN(32))
model.summary()
from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing import sequence
max_features = 10000
maxlen = 500
batch_size = 32
print('Loading data...')
(input_train, y_train), (input_test,
y_test) = imdb.load_data(num_words=max_features)
print(len(input_train), 'train sequences')
print(len(input_test), 'test sequences')
print('Pad sequences (samples x time)')
input_train = sequence.pad_sequences(input_train, maxlen=maxlen)
input_test = sequence.pad_sequences(input_test, maxlen=maxlen)
print('input_train shape:', input_train.shape)
print('input_test shape:', input_test.shape)
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, SimpleRNN
from tensorflow.keras.optimizers import RMSprop, SGD
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
n_classes = 10
n_classes = 10
x_train = x_train.reshape(-1,28,28)
x_test = x_test.reshape(-1,28,28)
y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)
# create and fit the SimpleRNN model
model = Sequential()
model.add(SimpleRNN(units=16, activation='relu', input_shape=(28,28)))
model.add(Dense(n_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=0.01), metrics=['accuracy'])
model.summary()
model.fit(x_train, y_train, batch_size=100, epochs=20)
score = model.evaluate(x_test, y_test)
print('\nTest loss:', score[0])
print('Test accuracy:', score[1])
x = data[i:i + T]
X.append(x)
y_temp = data[i + T]
y.append(y_temp)
# Since we need an N x T x D input
X = np.array(X).reshape(-1, T, D)
y = np.array(y)
print(X.shape)
print(y.shape)
N, T, D = X.shape
i_layer = Input(shape=(T, D))
h_layer = SimpleRNN(10)(i_layer)
o_layer = Dense(1)(h_layer)
model = Model(i_layer, o_layer)
model.compile(loss='mse', optimizer=Adam(lr=0.1))
index = -N // 2
report = model.fit(X[:index],
y[:index],
epochs=50,
validation_data=(X[index:], y[index:]))
plt.plot(report.history['loss'], label='training_loss')
plt.plot(report.history['val_loss'], label='validation_loss')
plt.legend()
y_test = y[index:]
def build_attention_model(self):
if self.cell_type == "RNN":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = SimpleRNN(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = SimpleRNN(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "LSTM":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state_h, state_c = encoder(encoder_outputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state_h, state_c]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
decoder_outputs, _, _ = decoder(
decoder_outputs, initial_state=encoder_states
)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "GRU":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = GRU(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = GRU(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
def build_configurable_model(self):
if self.cell_type == "RNN":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = SimpleRNN(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = SimpleRNN(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "LSTM":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state_h, state_c = encoder(encoder_outputs)
encoder_states = [state_h, state_c]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
decoder_outputs, _, _ = decoder(
decoder_outputs, initial_state=encoder_states
)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "GRU":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = GRU(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = GRU(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
import numpy as np
# 시계열 데이터
x = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]])
y = np.array([4, 5, 6, 7])
print('x.shape: ', x.shape) # (4, 3)
print('y.shape: ', y.shape) # (4,)
x = x.reshape(4, 3, 1)
# 2. 모델구성
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, SimpleRNN
model = Sequential()
model.add(SimpleRNN(10, activation='relu',
input_shape=(3, 1))) # (3, 1) -> (timesteps, input_dim)
model.add(Dense(20))
model.add(Dense(10))
model.add(Dense(1))
# LSTM - 3차원, Dense - 2차원
model.summary()
'''
통상적으로 LSTM이 더 좋다
LSTM이 연산을 더 많이 하기 떄문이다
gate는 존재하지 않기 때문에
param은 120으로 (n + m + 1) * m 으로 계산
activation='tanh'
입력 데이터가 커지면 학습능력이 저하 (앞쪽 데이터의 영향력이 줄어든다)
# train_x.shape => (samples, timesteps)
# train_y.shape => (samples)
train_x, train_y = split_sequence(train_y, step=n_timesteps)
print("shape x:{} / y:{}".format(train_x.shape, train_y.shape))
# RNN 입력 벡터 크기를 맞추기 위해 벡터 차원 크기 변경
# reshape from [samples, timesteps] into [samples, timesteps, features]
train_x = train_x.reshape(train_x.shape[0], train_x.shape[1], n_features)
print("train_x.shape = {}".format(train_x.shape))
print("train_y.shape = {}".format(train_y.shape))
# RNN 모델 정의
model = Sequential()
model.add(
SimpleRNN(units=10,
return_sequences=False,
input_shape=(n_timesteps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# 모델 학습
np.random.seed(0)
from tensorflow.keras.callbacks import EarlyStopping
early_stopping = EarlyStopping(monitor='loss', patience=5, mode='auto')
history = model.fit(train_x, train_y, epochs=1000, callbacks=[early_stopping])
# loss 그래프 생성
plt.plot(history.history['loss'], label="loss")
plt.legend(loc="upper right")
plt.show()
def _compile_model(self):
self.model = Sequential()
self.model.add(SimpleRNN(self.d, input_shape=(None, self.d),
activation=None, kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer))
self.model.compile(**self.compile_opts)
# 1. 데이터
import numpy as np
x = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]])
y = np.array([4, 5, 6, 7])
print(x.shape, y.shape) #(4,3) (4,)
#x = x.reshape(4, 3, 1) # LSTM에 넣기 위해서 3차원으로 변환, Dense는 2차원 참고로 CNN은 4차원
x = x.reshape(4, 3, 1) # -1은 제일끝, -2는 끝에서 두번째를 의미 (-1, 3, 1) (4, -2, 1)
# 2. 모델구성
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, SimpleRNN
model = Sequential()
model.add(SimpleRNN(10, activation='relu',
input_shape=(3, 1))) #(행, 열, 몇개씩자르는지)
model.add(Dense(20))
model.add(Dense(20))
model.add(Dense(10))
model.add(Dense(1))
model.summary()
# param# 120 Gate 없어
# 3. 컴파일, 훈련
model.compile(loss='mse', optimizer='adam')
model.fit(x, y, epochs=100, batch_size=1)
# 4. 평가, 예측
loss = model.evaluate(x, y, batch_size=1)
test_out_ints = enc.encode(test_out) test_out_enc = to_categorical(test_out_ints) trainX, trainY = convertToMatrix(train_in, train_out_enc, step) testX, testY = convertToMatrix(test_in, test_out_enc, step) #Finally, we'll reshape trainX and testX to fit with the Keras model. RNN model requires three-dimensional input data. You can see the shape of testX below. trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1])) trainX.shape # SimpleRNN model model = Sequential() model.add(SimpleRNN(units=8, input_shape=(1, step), activation="relu")) model.add(Dense(8, activation="relu")) model.add(Dense(NUM_LABELS, activation="softmax")) model.compile(loss='categorical_crossentropy', optimizer='rmsprop') model.summary() model.fit(trainX, trainY, epochs=50, batch_size=4, verbose=2) testPredict = model.predict_classes(testX) #Next, we'll check the loss trainScore = model.evaluate(trainX, trainY, verbose=0) print(trainScore) #Finally, we check the result in a plot. A vertical line in a plot identifies a splitting point between the training and the test part.
#打乱顺序
np.random.seed(7)
np.random.shuffle(x_train)
np.random.seed(7)
np.random.shuffle(y_train)
tf.random.set_seed(7)
# 使x_train符合Embedding输入要求：[送入样本数， 循环核时间展开步数] ，
# 此处整个数据集送入所以送入，送入样本数为len(x_train)；输入4个字母出结果，循环核时间展开步数为4。
x_train = np.reshape(x_train, (len(x_train), 4))
y_train = np.array(y_train)
model = tf.keras.Sequential([
Embedding(26, 2),
SimpleRNN(10), #搭建具有三个记忆体的循环层(记忆体个数越多，记忆越好，但占用资源越多)
Dense(26, activation='softmax') #实现了输出层的计算（由于要映射到独热码编码，找到输出概率最大的字母，所以为5）
])
model.compile(
optimizer=tf.keras.optimizers.Adam(0.01),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['sparse_categorical_accuracy'])
checkpoint_save_path = "./checkpoint/rnn_embedding_4.ckpt"
if os.path.exists(checkpoint_save_path + '.index'):
print('----load the model-----')
model.load_weights(checkpoint_save_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_save_path,
save_weights_only=True,
save_best_only=True,
y_train.append(training_set_scaled[i])
np.random.seed(7)
np.random.shuffle(x_train)
np.random.seed(7)
np.random.shuffle(y_train)
tf.random.set_seed(7)
# 使x_train符合Embedding输入要求：[送入样本数， 循环核时间展开步数] ，
# 此处整个数据集送入所以送入，送入样本数为len(x_train)；输入4个字母出结果，循环核时间展开步数为4。
x_train = np.reshape(x_train, (len(x_train), 4))
y_train = np.array(y_train)
model = tf.keras.Sequential([
Embedding(26, 2), # 2表示每个单词会用两个数值编码，该层会生成一个26行2列的矩阵，实现编码可训练
SimpleRNN(10),
Dense(26, activation='softmax')
])
model.compile(
optimizer=tf.keras.optimizers.Adam(0.01),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['sparse_categorical_accuracy'])
checkpoint_save_path = "./checkpoint/rnn_embedding_4pre1.ckpt"
if os.path.exists(checkpoint_save_path + '.index'):
print('-------------load the model-----------------')
model.load_weights(checkpoint_save_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(
#1. 데이터
x = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6], [5, 6, 7], [6, 7, 8],
[7, 8, 9], [8, 9, 10], [9, 10, 11], [10, 11, 12], [20, 30, 40],
[30, 40, 50], [40, 50, 60]])
y = np.array([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 50, 60, 70])
print("x : shape", x.shape) # (13, 3)
print("y : shape", y.shape) # (13,)
x = x.reshape(13, 3, 1)
#2. 모델구성(LSTM)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, SimpleRNN
model = Sequential()
model.add(SimpleRNN(13, activation='linear', input_shape=(3, 1)))
model.add(Dense(26))
model.add(Dense(52))
model.add(Dense(13))
model.add(Dense(13))
model.add(Dense(1))
model.summary()
#3.컴파일, 훈련
from tensorflow.keras.callbacks import EarlyStopping
early_stopping = EarlyStopping(monitor='loss', patience=30, mode='auto')
model.compile(loss='mse', optimizer='adam', metrics=['acc'])
model.fit(x, y, epochs=1000, batch_size=1, callbacks=[early_stopping])
#4. 평가, 예측
loss = model.evaluate(x, y)
from tensorflow.keras.layers import Embedding, Dense, LSTM, SimpleRNN
# 각 단어의 임베딩 벡터는 10차원을 가지고, 128의 은닉 상태 크기를 가지는 LSTM을 사용
# model = Sequential()
# model.add(Embedding(vocab_size, 10, input_length=max_len-1)) # y데이터를 분리하였으므로 이제 X데이터의 길이는 기존 데이터의 길이 - 1
# model.add(LSTM(128))
# model.add(Dense(vocab_size, activation='softmax'))
# model.summary()
#
# model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# model.fit(X, y, epochs=200, verbose=2)
model = Sequential()
model.add(Embedding(vocab_size, 10,
input_length=max_len - 1)) # 레이블을 분리하였으므로 이제 X의 길이는 5
model.add(SimpleRNN(128))
model.add(Dense(vocab_size, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X, y, epochs=200, verbose=2)
print("\n Test Accuracy: %.4f" %
(model.evaluate(X, y)[1])) # Test Accuracy: 0.9246
# sentence_generation을 만들어서 문장을 생성
def sentence_generation(model, t, current_word, n): # 모델, 토크나이저, 현재 단어, 반복할 횟수
init_word = current_word # 처음 들어온 단어도 마지막에 같이 출력하기위해 저장
class SimpleRNNInference:
def __init__(self):
pass
def __call__(self):
pass
def forward_step(self):
pass
# data set shape = (num_samples, num_timesteps, num_features)
# input shape = (num_timesteps, num_features)
# If return_sequences == True:
# output shape = (num_timesteps, units)
# Else:
# output shape = (1, units)
x = np.random.normal(size=(1, 3, 2))
units = 4
return_sequences = False
model = Sequential()
model.add(
SimpleRNN(units=units,
return_sequences=return_sequences,
input_shape=x.shape[1:]))
model.compile(loss="mse", optimizer="Adam")
model.summary()
def build_neural_network_model(Recurrent_Neural_Network: List[Any],
n_inputs: int, n_days: int) -> Sequential:
"""
Builds neural net from config_neural_network_models.py
Parameters
----------
Recurrent_Neural_Network: List[Any]
List of layers with parameters as a dictionary in the file
n_inputs: int
Number of days that will be fed into the NN
n_days: int
Number of days the NN wants to predict
Returns
-------
model: Sequential
Keras sequential model with layers from the file
"""
model = Sequential()
for idx_layer, d_layer in enumerate(Recurrent_Neural_Network):
# Recurrent Neural Network
if str(*d_layer) == "SimpleRNN":
# Is this the input layer? If so, define input_shape
if idx_layer == 0:
model.add(
SimpleRNN(**d_layer["SimpleRNN"],
input_shape=(n_inputs, 1)))
# Is this the last output layer? If so, set units to prediction days
elif idx_layer == (len(Recurrent_Neural_Network) - 1):
model.add(SimpleRNN(**d_layer["SimpleRNN"], units=n_days))
else:
model.add(SimpleRNN(**d_layer["SimpleRNN"]))
# Long-Short Term-Memory
elif str(*d_layer) == "LSTM":
# Is this the input layer? If so, define input_shape
if idx_layer == 0:
model.add(LSTM(**d_layer["LSTM"], input_shape=(n_inputs, 1)))
# Is this the last output layer? If so, set units to prediction days
elif idx_layer == (len(Recurrent_Neural_Network) - 1):
model.add(LSTM(**d_layer["LSTM"], units=n_days))
else:
model.add(LSTM(**d_layer["LSTM"]))
# Dense (Simple Neuron)
elif str(*d_layer) == "Dense":
# Is this the input layer? If so, define input_shape
if idx_layer == 0:
model.add(Dense(**d_layer["Dense"], input_dim=n_inputs))
# Is this the last output layer? If so, set units to prediction days
elif idx_layer == (len(Recurrent_Neural_Network) - 1):
model.add(Dense(**d_layer["Dense"], units=n_days))
else:
model.add(Dense(**d_layer["Dense"]))
# Conv1D Layer
elif str(*d_layer) == "Conv1D":
if idx_layer == 0:
model.add(
Conv1D(**d_layer["Conv1D"], input_shape=(n_inputs, 1)))
else:
model.add(Conv1D(**d_layer["Conv1D"]))
# Max Pooling Layer for after Conv Layer
elif str(*d_layer) == "MaxPool1D":
model.add(MaxPool1D(**d_layer["MaxPool1D"]))
# Allow for if user wants to do average pooling
elif str(*d_layer) == "AvgPool1D":
model.add(AvgPool1D(**d_layer["AvgPool1D"]))
# Dropout (Regularization)
elif str(*d_layer) == "Dropout":
model.add(Dropout(**d_layer["Dropout"]))
# Flatten layer for Convolutions
elif str(*d_layer) == "Flatten":
model.add(Flatten())
else:
print(f"Incorrect neuron type: {str(*d_layer)}")
return model
y_train = [
w_to_id['e'], w_to_id['a'], w_to_id['b'], w_to_id['c'], w_to_id['d']
]
np.random.seed(7)
np.random.shuffle(x_train)
np.random.seed(7)
np.random.shuffle(y_train)
tf.random.set_seed(7)
# 使x_train符合SimpleRNN输入要求：[送入样本数， 循环核时间展开步数， 每个时间步输入特征个数]。
# 此处整个数据集送入，送入样本数为len(x_train)；输入4个字母出结果，循环核时间展开步数为4; 表示为独热码有5个输入特征，每个时间步输入特征个数为5
x_train = np.reshape(x_train, (len(x_train), 4, 5))
y_train = np.array(y_train)
model = tf.keras.Sequential([SimpleRNN(3), Dense(5, activation='softmax')])
model.compile(
optimizer=tf.keras.optimizers.Adam(0.01),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['sparse_categorical_accuracy'])
checkpoint_save_path = "./checkpoint/rnn_onehot_4pre1.ckpt"
if os.path.exists(checkpoint_save_path + '.index'):
print('------------load the model---------------')
model.load_weights(checkpoint_save_path)
cp_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_save_path,
save_weights_only=True,
D = 1
Y = []
for t in range(len(series) - T):
x = series[t:t + T]
X.append(x)
y = series[t + T]
Y.append(y)
X = np.array(X).reshape(-1, T, 1) #N*T*D
Y = np.array(Y)
N = len(X)
print(X.shape, Y.shape)
#Build the model...................
i = Input(shape=(T, 1))
x = SimpleRNN(5, activation='relu')(i)
x = Dense(1)(x)
model = Model(i, x)
model.compile(loss='mse', optimizer=Adam(lr=0.1))
r = model.fit(
X[:-N // 2],
Y[:-N // 2],
epochs=50,
validation_data=(X[-N // 2:], Y[-N // 2:]),
)
#Plot the loss...............
plt.plot(r.history['loss'], label='loss')
plt.plot(r.history['val_loss'], label='val_loss')
plt.legend()
