# conver 2d dataframe to 3d array X_train_ar = X_train.to_numpy() # RNN input is a 3d tensor and hence X_test_ar = X_test.to_numpy() # convert numpy 2d dataframe to array first X_train_3d = X_train_ar[:,newaxis,:] # RNN input is a 3d tensor and hence X_test_3d = X_test_ar[:,newaxis,:] # convert 2d to 3d array using newaxis print(X_train_3d.shape) print(X_test_3d.shape) print(y_train.shape) print(y_test.shape) # RNN model = Sequential() model.add(SimpleRNN(5, input_shape=(X_train_3d.shape[1:]), return_sequences=True)) model.add(Dense(1, activation='softmax')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(X_train_3d, y_train, epochs=30) # LSTM model = Sequential() model.add(LSTM(5, input_shape=(X_train_3d.shape[1:]), return_sequences=True)) model.add(Dense(1, activation='softmax')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(X_train_3d, y_train, epochs=30) # In[ ]:
X = np.zeros((len(input_chars), SEQLEN, nb_chars), dtype=np.bool)#5000 x 10 x 55
y = np.zeros((len(input_chars), nb_chars), dtype=np.bool)
for i, input_char in enumerate(input_chars):
for j, ch in enumerate(input_char):
X[i, j, char2index[ch]] = 1
y[i, char2index[label_chars[i]]] = 1
HIDDEN_SIZE = 128
BATCH_SIZE = 128
NUM_ITERATIONS = 25
NUM_EPOCHS_PER_ITERATION = 1
NUM_PREDS_PER_EPOCH = 100
model = Sequential()
model.add(SimpleRNN(HIDDEN_SIZE, return_sequences=False,input_shape=(SEQLEN, nb_chars),unroll=True))
model.add(Dense(nb_chars))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
model.summary()
for iteration in range(NUM_ITERATIONS):
print("=" * 50)
print("Iteration #: %d" % (iteration))
model.fit(X, y, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS_PER_ITERATION)
#test_idx = np.random.randint(len(input_chars))
test_chars = "are and hi"
print("Generating from seed: %s" % (test_chars))
print(test_chars, end="")
for i in range(NUM_PREDS_PER_EPOCH):
def test_SimpleRNN(self):
# all recurrent layers inherit output_shape
# from the same base recurrent layer
layer = SimpleRNN(3, 2)
input_data = np.random.random((2, 2, 3))
check_layer_output_shape(layer, input_data)
def build(self):
input_shape = (self.frames_n, self.num_features)
self.model = Sequential()
for i in range(self.num_rnn_layers):
return_sequences = True
if i == self.num_rnn_layers - 1:
return_sequences = False
specify_input_shape = False
if i == 0:
specify_input_shape = True
name = 'rnn-' + str(i)
if self.use_gru:
if self.is_bidirectional:
if specify_input_shape:
self.model.add(
Bidirectional(GRU(
self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name),
input_shape=input_shape,
merge_mode='concat'))
else:
self.model.add(
Bidirectional(GRU(
self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name),
merge_mode='concat'))
else:
if specify_input_shape:
self.model.add(
GRU(self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name,
input_shape=input_shape))
else:
self.model.add(
GRU(self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name))
else:
if self.is_bidirectional:
if specify_input_shape:
self.model.add(
Bidirectional(SimpleRNN(
self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name),
input_shape=input_shape,
merge_mode='concat'))
else:
self.model.add(
Bidirectional(SimpleRNN(
self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name),
merge_mode='concat'))
else:
if specify_input_shape:
self.model.add(
SimpleRNN(self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name,
input_shape=input_shape))
else:
self.model.add(
SimpleRNN(self.num_neurons_in_rnn_layer,
return_sequences=return_sequences,
name=name))
if self.dropout > 0.0:
self.model.add(Dropout(self.dropout))
for i in range(self.num_output_dense_layers):
name = 'dense-' + str(i)
self.model.add(
Dense(self.num_neurons_in_output_dense_layers,
activation=self.activation_output_dense_layers,
name=name))
self.model.add(
Dense(self.num_classes, name='softmax', activation='softmax'))
def build(self,
dafm_type="dafm-afm",
optimizer="rmsprop",
learning_rate=0.01,
activation="linear",
Q_jk_initialize=0,
section="",
section_count=0,
model1="",
stateful=False,
theta_student="False",
student_count=0,
binary="False"):
skills = np.shape(Q_jk_initialize)[1]
steps = np.shape(Q_jk_initialize)[0]
self.activation = activation
if '-' in self.activation:
activation = self.custom_activation
if dafm_type.split("_")[-1] == "different":
skills = int(float(dafm_type.split("_")[-2]) * skills)
dafm_type = dafm_type.split('_')[0]
if dafm_type.split("_")[0] == "round-fine-tuned":
try:
self.round_threshold = float(dafm_type.split("_")[-1])
dafm_type = dafm_type.split("_")[0]
except:
pass
q_jk_size = skills
if '^' in dafm_type:
q_jk_size = skills
skills = int(float(dafm_type.split('^')[-1]) * skills)
dafm_type = dafm_type.split('^')[0]
self.dafm_type = dafm_type
if dafm_type == "random-uniform" or dafm_type == "random-normal":
qtrainable, finetuning, randomize = True, False, True
self.random_init = dafm_type.split('-')[-1]
elif dafm_type == "dafm-afm":
qtrainable, finetuning, randomize = False, False, False
elif dafm_type == "fine-tuned":
qtrainable, finetuning, randomize = True, True, False
elif dafm_type == "kcinitialize":
qtrainable, finetuning, randomize = True, False, False
elif dafm_type == "round-fine-tuned":
# if not self.round_threshold == -1:
# rounded_Qjk = np.abs(Q_jk1 - Q_jk_initialize)
# Q_jk1[rounded_Qjk <= self.round_threshold] = Q_jk_initialize[rounded_Qjk <= self.round_threshold]
# Q_jk1[rounded_Qjk > self.round_threshold] = np.ones(np.shape(Q_jk_initialize[rounded_Qjk > self.round_threshold])) - Q_jk_initialize[rounded_Qjk > self.round_threshold]
# else:
Q_jk1 = model1.get_layer("Q_jk").get_weights()[0]
Q_jk1 = np.minimum(
np.ones(np.shape(Q_jk1)),
np.maximum(np.round(Q_jk1), np.zeros(np.shape(Q_jk1))))
model1.get_layer("Q_jk").set_weights([Q_jk1])
return model1
elif dafm_type == "qjk-dense":
qtrainable, finetuning, randomize = False, False, False
activation_dense = activation
elif dafm_type == "random-qjk-dense-normal" or dafm_type == "random-qjk-dense-uniform":
qtrainable, finetuning, randomize = False, False, True
self.random_init = dafm_type.split('-')[-1]
activation_dense = activation
else:
print("No Valid Model Found")
sys.exit()
if section == "onehot":
section_input = Input(batch_shape=(None, None, section_count),
name='section_input')
if not theta_student == "False":
student_input = Input(batch_shape=(None, None, student_count),
name='student_input')
virtual_input1 = Input(batch_shape=(None, None, 1),
name='virtual_input1')
if finetuning:
B_k = TimeDistributed(Dense(
skills,
activation='linear',
kernel_initializer=self.f(
model1.get_layer("B_k").get_weights()[0]),
use_bias=False),
name="B_k")(virtual_input1)
T_k = TimeDistributed(Dense(
skills,
activation='linear',
kernel_initializer=self.f(
model1.get_layer("T_k").get_weights()[0]),
use_bias=False),
name="T_k")(virtual_input1)
bias_layer = TimeDistributed(Dense(
1,
activation='linear',
use_bias=False,
kernel_initializer=self.f(
model1.get_layer("bias").get_weights()[0]),
trainable=True),
name="bias")(virtual_input1)
else:
B_k = TimeDistributed(Dense(skills,
activation='linear',
use_bias=False,
trainable=True),
name="B_k")(virtual_input1)
T_k = TimeDistributed(Dense(skills,
activation='linear',
use_bias=False,
trainable=True),
name="T_k")(virtual_input1)
bias_layer = TimeDistributed(Dense(
1,
activation='linear',
use_bias=False,
kernel_initializer=initializers.Zeros(),
trainable=True),
name="bias")(virtual_input1)
step_input = Input(batch_shape=(None, None, steps), name='step_input')
if randomize:
if binary == "False":
Q_jk = TimeDistributed(Dense(
q_jk_size,
use_bias=False,
activation=activation,
kernel_initializer=self.custom_random),
trainable=qtrainable,
name="Q_jk")(step_input)
else:
Q_jk = TimeDistributed(BinaryDense(
q_jk_size,
use_bias=False,
activation=activation,
kernel_initializer=self.custom_random),
trainable=qtrainable,
name="Q_jk")(step_input)
else:
if binary == "False":
Q_jk = TimeDistributed(Dense(
skills,
activation=activation,
kernel_initializer=self.f(Q_jk_initialize),
use_bias=False,
trainable=qtrainable),
trainable=qtrainable,
name="Q_jk")(step_input)
else:
Q_jk = TimeDistributed(BinaryDense(
skills,
activation=activation,
kernel_initializer=self.f(Q_jk_initialize),
trainable=qtrainable,
use_bias=False),
name="Q_jk",
trainable=qtrainable)(step_input)
if dafm_type == "random-qjk-dense-normal" or dafm_type == "random-qjk-dense-uniform":
if binary == "False":
Q_jk = TimeDistributed(Dense(
skills,
activation=activation_dense,
use_bias=False,
kernel_initializer=self.custom_random,
trainable=True),
name="Q_jk_dense")(Q_jk)
else:
Q_jk = TimeDistributed(BinaryDense(
skills,
activation=activation_dense,
use_bias=False,
kernel_initializer=self.custom_random,
trainable=True),
name="Q_jk_dense")(Q_jk)
elif dafm_type == "qjk-dense":
if binary == 'False':
Q_jk = TimeDistributed(Dense(
skills,
activation=activation_dense,
use_bias=False,
kernel_initializer=initializers.Identity(),
trainable=True),
name="Q_jk_dense")(Q_jk)
else:
Q_jk = TimeDistributed(BinaryDense(
skills,
activation=activation_dense,
use_bias=False,
kernel_initializer=initializers.Identity(),
trainable=True),
name="Q_jk_dense")(Q_jk)
else:
pass
Qjk_mul_Bk = multiply([Q_jk, B_k])
sum_Qjk_Bk = TimeDistributed(Dense(
1,
activation='linear',
trainable=False,
kernel_initializer=initializers.Ones(),
use_bias=False),
trainable=False,
name="sum_Qjk_Bk")(Qjk_mul_Bk)
P_k = SimpleRNN(skills,
kernel_initializer=initializers.Identity(),
recurrent_initializer=initializers.Identity(),
use_bias=False,
trainable=False,
activation='linear',
return_sequences=True,
name="P_k")(Q_jk)
Qjk_mul_Pk_mul_Tk = multiply([Q_jk, P_k, T_k])
sum_Qjk_Pk_Tk = TimeDistributed(
Dense(1,
activation='linear',
trainable=False,
kernel_initializer=initializers.Ones(),
use_bias=False),
trainable=False,
name="sum_Qjk_Pk_Tk")(Qjk_mul_Pk_mul_Tk)
Concatenate = concatenate([bias_layer, sum_Qjk_Bk, sum_Qjk_Pk_Tk])
if not (theta_student == "False"):
if finetuning:
theta = TimeDistributed(Dense(
1,
activation="linear",
use_bias=False,
kernel_initializer=self.f(
model1.get_layer("theta").get_weights()[0])),
name='theta')(student_input)
else:
theta = TimeDistributed(Dense(1,
activation="linear",
use_bias=False),
name='theta')(student_input)
Concatenate = concatenate([Concatenate, theta])
if section == "onehot":
if finetuning:
S_k = TimeDistributed(Dense(
1,
activation="linear",
use_bias=False,
kernel_initializer=self.f(
model1.get_layer("S_k").get_weights()[0])),
name='S_k')(section_input)
else:
S_k = TimeDistributed(Dense(1,
activation="linear",
use_bias=False),
name='S_k')(section_input)
Concatenate = concatenate([Concatenate, S_k])
output = TimeDistributed(Dense(1,
activation="sigmoid",
trainable=False,
kernel_initializer=initializers.Ones(),
use_bias=False),
trainable=False,
name="output")(Concatenate)
if section == "onehot" and not (theta_student == "False"):
model = Model(inputs=[
virtual_input1, step_input, section_input, student_input
],
outputs=output)
elif section == "onehot" and theta_student == "False":
model = Model(inputs=[virtual_input1, step_input, section_input],
outputs=output)
elif not (section == "onehot") and not (theta_student == "False"):
model = Model(inputs=[virtual_input1, step_input, student_input],
outputs=output)
else:
model = Model(inputs=[virtual_input1, step_input], outputs=output)
d_optimizer = {
"rmsprop": optimizers.RMSprop(lr=learning_rate),
"adam": optimizers.Adam(lr=learning_rate),
"adagrad": optimizers.Adagrad(lr=learning_rate)
}
model.compile(optimizer=d_optimizer[optimizer], loss=self.custom_bce)
return model
def rnn_input(inputs, N, spad, dropout=3/4, dropoutfix_inp=0, dropoutfix_rec=0,
sdim=2, rnnbidi=True, return_sequences=False,
rnn=GRU, rnnact='tanh', rnninit='glorot_uniform', rnnbidi_mode='sum',
rnnlevels=1, pfx=''):
""" An RNN layer that takes sequence of embeddings e0, e1 and
processes them using an RNN + dropout.
If return_sequences=False, it returns just the final hidden state of the RNN;
otherwise, it return a sequence of contextual token embeddings instead.
At any rate, the output layers are e0s_, e1s_.
If rnnlevels>1, a multi-level stacked RNN architecture like in Wang&Nyberg
http://www.aclweb.org/anthology/P15-2116 is applied, however with skip-connections
i.e. the inner RNNs have both the outer RNN and original embeddings as inputs.
"""
deep_inputs = inputs
linear_rnn = Activation('linear')
for i in range(1, rnnlevels):
sequences = rnn_input(deep_inputs, N, spad, dropout=0, sdim=sdim, rnnbidi=rnnbidi, return_sequences=True,
rnn=rnn, rnnact=rnnact, rnninit=rnninit, rnnbidi_mode=rnnbidi_mode,
rnnlevels=1, inputs=deep_inputs, pfx=pfx+'L%d'%(i,))
inputs = deep_inputs
#model.add_node(name=pfx+'L%de0s_j'%(i,), inputs=[inputs[0], pfx+'L%de0s_'%(i,)], merge_mode='concat', layer=Activation('linear'))
#model.add_node(name=pfx+'L%de1s_j'%(i,), inputs=[inputs[1], pfx+'L%de1s_'%(i,)], merge_mode='concat', layer=Activation('linear'))
#deep_inputs = ['L%de0s_j'%(i,), 'L%de1s_j'%(i,)]
e0s_j = linear_rnn(concatenate([inputs[0], sequences[0]]))
e1s_j = linear_rnn(concatenate([inputs[1], sequences[1]]))
deep_inputs = [e0s_j, e1s_j]
if rnnbidi:
if rnnbidi_mode == 'concat':
sdim /= 2
#model.add_shared_node(name=pfx+'rnnf', inputs=deep_inputs, outputs=[pfx+'e0sf', pfx+'e1sf'],
# layer=rnn(input_dim=N, output_dim=int(N*sdim), input_length=spad,
# init=rnninit, activation=rnnact,
# return_sequences=return_sequences,
# dropout_W=dropoutfix_inp, dropout_U=dropoutfix_rec))
#model.add_shared_node(name=pfx+'rnnb', inputs=deep_inputs, outputs=[pfx+'e0sb', pfx+'e1sb'],
# layer=rnn(input_dim=N, output_dim=int(N*sdim), input_length=spad,
# init=rnninit, activation=rnnact,
# return_sequences=return_sequences, go_backwards=True,
# dropout_W=dropoutfix_inp, dropout_U=dropoutfix_rec))
rnnf = SimpleRNN(int(N*sdim), activation=rnnact, dropout=dropoutfix_inp,
recurrent_dropout=dropoutfix_rec, return_sequences=return_sequences,
name='rnnf')
e0sf = rnnf(deep_inputs[0])
e1sf = rnnf(deep_inputs[1])
rnnb = SimpleRNN(int(N*sdim), activation=rnnact, dropout=dropoutfix_inp,
recurrent_dropout=dropoutfix_rec, return_sequences=return_sequences,
go_backwards=True, name='rnnb')
e0sb = rnnb(deep_inputs[0])
e1sb = rnnb(deep_inputs[1])
#model.add_node(name=pfx+'e0s', inputs=[pfx+'e0sf', pfx+'e0sb'], merge_mode=rnnbidi_mode, layer=Activation('linear'))
#model.add_node(name=pfx+'e1s', inputs=[pfx+'e1sf', pfx+'e1sb'], merge_mode=rnnbidi_mode, layer=Activation('linear'))
e0s = linear_rnn(add([e0sf, e0sb]))
e1s = linear_rnn(add([e1sf, e1sb]))
else:
#model.add_shared_node(name=pfx+'rnn', inputs=deep_inputs, outputs=[pfx+'e0s', pfx+'e1s'],
# layer=rnn(input_dim=N, output_dim=int(N*sdim), input_length=spad,
# init=rnninit, activation=rnnact,
# return_sequences=return_sequences,
# dropout_W=dropoutfix_inp, dropout_U=dropoutfix_rec))
rnn = SimpleRNN(int(N*sdim), activation=rnnact, dropout=dropoutfix_inp,
recurrent_dropout=dropoutfix_rec, return_sequences=return_sequences,
name='rnn')
e0s = SimpleRNN(deep_inputs[0])
e1s = SimpleRNN(deep_inputs[1])
#model.add_shared_node(name=pfx+'rnndrop', inputs=[pfx+'e0s', pfx+'e1s'], outputs=[pfx+'e0s_', pfx+'e1s_'],
# layer=Dropout(dropout, input_shape=(spad, int(N*sdim)) if return_sequences else (int(N*sdim),)))
rnndrop = Dropout(dropout, input_shape=(spad, int(N*sdim)) if return_sequences else (int(N*sdim),), name='rnndrop')
e0s_ = rnndrop(e0s)
e1s_ = rnndrop(e1s)
return [e0s_, e1s_]
def DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=None,
weight_decay=1E-4):
""" Build the DenseNet model"""
model_input = Input(shape=img_dim)
def lambda_output(input_shape):
return (input_shape[0], 1, input_shape[1])
Expand = Lambda(lambda x: K.expand_dims(x, 1), output_shape=lambda_output)
assert (depth - 4) % 3 == 0, "Depth must be 3 N + 4"
# layers in each dense block
nb_layers = int((depth - 4) / 3)
# Initial convolution
x = Convolution2D(nb_filter,
3,
3,
init="he_uniform",
border_mode="same",
name="initial_conv2D",
bias=False,
W_regularizer=l2(weight_decay))(model_input)
list_RNN_input = []
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
x, nb_filter = denseblock(x,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# add transition
x = transition(x,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x_RNN = Convolution2D(1,
3,
3,
init="he_uniform",
border_mode="same",
bias=False,
subsample=(2 - block_idx, 2 - block_idx),
W_regularizer=l2(weight_decay))(x)
x_RNN = Flatten()(x_RNN)
x_RNN = Expand(x_RNN)
list_RNN_input.append(x_RNN)
# The last denseblock does not have a transition
x, nb_filter = denseblock(x,
nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(mode=0,
axis=1,
gamma_regularizer=l2(weight_decay),
beta_regularizer=l2(weight_decay))(x)
x = Activation('relu')(x)
x_RNN = Convolution2D(1,
3,
3,
init="he_uniform",
border_mode="same",
bias=False,
W_regularizer=l2(weight_decay))(x)
x_RNN = Flatten()(x_RNN)
x_RNN = Expand(x_RNN)
list_RNN_input.append(x_RNN)
x_RNN = merge(list_RNN_input, mode='concat', concat_axis=1)
x_RNN = SimpleRNN(100)(x_RNN)
x = GlobalAveragePooling2D()(x)
x = Flatten()(x)
x = merge([x, x_RNN], mode="concat")
x = Dense(nb_classes,
activation='softmax',
W_regularizer=l2(weight_decay),
b_regularizer=l2(weight_decay))(x)
densenet = Model(input=[model_input], output=[x], name="DenseNet")
return densenet
#make prediction
trainPredict = model.predict(X_train)
testPredict = model.predict(X_test)
print testPredict
print y_test
print label_tranform(testPredict)
accuracy = accuracy_score(y_test, label_tranform(testPredict))
print "Accuuracy LSTM", accuracy
model = Sequential()
model.add(
SimpleRNN(4,
input_shape=X_train.shape[1:],
W_regularizer=l2(0.01),
dropout_W=0.4,
dropout_U=0.4,
U_regularizer=l2(0.4),
b_regularizer=l2(0.4)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(X_train, y_train, nb_epoch=100, batch_size=1, verbose=2)
#make prediction
trainPredict = model.predict(X_train)
testPredict = model.predict(X_test)
print testPredict
print y_test
print label_tranform(testPredict)
accuracy = accuracy_score(y_test, label_tranform(testPredict))
print "Accuuracy RNN", accuracy
n_in = 1
n_out = 1
num_neurons = 1
n_hidden = 200
# ニューラルネットワークを重ねてモデルを構成。
model = Sequential()
# 入力層から再帰層までの定義(SimpleRNNNクラス)
## batch_input_shape: (バッチ数,学習データのステップ数、説明変数の数)を多プルで指定。
## return_sequences: Falseの場合は、最後の時刻のみの出力を得る。
model.add(SimpleRNN(n_hidden, batch_input_shape=(None, affect_length, num_neurons), return_sequences=False))
# 再帰層から出力層までの定義(Dense,Activation)
model.add(Dense(num_neurons)) # Dense == 全結合モデル。 num_neurons: ユニット数
model.add(Activation('linear')) # 活性化関数を指定。(linear関数)
optimizer = Adam(lr = 0.001) # 学習率: 0.001。 #Adam: 最適化手法の一つ。デファクトスタンダートとして広く使われる。
return stacked.dimshuffle((0, 2, 1))
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, TimeDistributedDense, RepeatVector
from keras.layers.recurrent import SimpleRNN
from keras.optimizers import SGD
# (nsamples, ntimesteps, nfeatures)
model = Sequential()
if False:
model.add(Dense(bn.num_vars, input_dim=bn.num_vars, activation='tanh')) # , init='uniform', activation='sigmoid'))
model.add(RepeatVector(timesteps))
elif False:
model.add(Dense(hidden_dims, input_dim=bn.num_vars, activation='tanh')) # , init='uniform', activation='sigmoid'))
model.add(PadZerosVector(timesteps))
model.add(SimpleRNN(hidden_dims, return_sequences=True, input_length=timesteps, activation='tanh'))
model.add(TimeDistributedDense(bn.num_vars, activation='sigmoid'))
#model.add(Activation('sigmoid'))
elif True:
#activation = 'relu'
activation = 'tanh'
model.add(TimeDistributedDense(hidden_dims, input_dim=bn.num_vars, input_length=timesteps, activation=activation))
model.add(SimpleRNN(hidden_dims, return_sequences=True, input_length=timesteps, activation=activation))
model.add(TimeDistributedDense(bn.num_vars, activation='sigmoid'))
#model.add(Activation('sigmoid'))
else:
#from keras.layers.advanced_activations import ELU, LeakyReLU
#
model.add(SimpleRNN(hidden_dims, input_dim=bn.num_vars, return_sequences=True, input_length=timesteps, activation='tanh'))
model.add(TimeDistributedDense(bn.num_vars, activation='sigmoid'))
def build(self):
raw_current = Input(shape=(self.time_length,), dtype='int32')
if len(self.embeddings_matrix) == 0:
embedding = Embedding(input_dim=len(self.word2idx),
output_dim=self.embedding_size,
input_length=self.time_length)
else:
embedding = Embedding(len(self.embeddings_matrix),
self.embedding_size,
weights=[self.embeddings_matrix],
trainable=False)
current = embedding(raw_current)
# set optimizer
if self.update_f == 'sgd':
opt_func = SGD(lr=self.learning_rate,
momentum=self.momentum, decay=self.decay_rate)
elif self.update_f == 'rmsprop':
opt_func = RMSprop(lr=self.learning_rate,
rho=self.rho, epsilon=self.smooth_eps)
elif self.update_f == 'adagrad':
opt_func = Adagrad(lr=self.learning_rate, epsilon=self.smooth_eps)
elif self.update_f == 'adadelta':
opt_func = Adadelta(lr=self.learning_rate,
rho=self.rho, epsilon=self.smooth_eps)
elif self.update_f == 'adam':
opt_func = Adam(lr=self.learning_rate, beta_1=self.beta1, beta_2=self.beta2,
epsilon=self.smooth_eps)
elif self.update_f == 'adamax':
opt_func = Adamax(lr=self.learning_rate, beta_1=self.beta1, beta_2=self.beta2,
epsilon=self.smooth_eps)
else:
sys.stderr.write("Invalid optimizer.\n")
exit()
# Vallina RNN (LSTM, SimpleRNN, GRU)
# Bidirectional-RNN (LSTM, SimpleRNN, GRU)
if self.arch == 'lstm' or self.arch == 'rnn' or self.arch == 'gru' \
or self.arch == 'blstm' or self.arch == 'brnn' or self.arch == 'bgru':
if 'rnn' in self.arch:
forward = SimpleRNN(self.hidden_size, return_sequences=True, activation=self.activation,
kernel_initializer=self.init_type)(current)
backward = SimpleRNN(self.hidden_size, return_sequences=True, activation=self.activation,
go_backwards=True, kernel_initializer=self.init_type)(current)
elif 'gru' in self.arch:
forward = GRU(self.hidden_size, return_sequences=True, init=self.init_type,
activation=self.activation)(current)
backward = GRU(self.hidden_size, return_sequences=True, init=self.init_type,
activation=self.activation, go_backwards=True)(current)
elif 'lstm' in self.arch:
forward = LSTM(self.hidden_size,
return_sequences=True,
activation=self.activation,
kernel_initializer=self.init_type)(current)
backward = LSTM(self.hidden_size,
return_sequences=True,
activation=self.activation,
go_backwards=True,
kernel_initializer=self.init_type)(current)
if 'b' in self.arch:
tagger = layers.concatenate([forward, backward])
else:
tagger = forward
if self.dropout:
tagger = Dropout(self.dropout_ratio)(tagger)
prediction = TimeDistributed(
Dense(self.output_vocab_size, activation='softmax'))(tagger)
self.model = Model(inputs=raw_current, outputs=prediction)
self.model.compile(
loss='categorical_crossentropy', optimizer=opt_func)
# 2-Stacked Layered RNN (LSTM, SimpleRNN, GRU)
elif self.arch == '2lstm' or self.arch == '2rnn' or self.arch == '2gru':
model = Sequential()
model.add(embedding)
if self.arch == '2lstm':
basic_model = LSTM(self.hidden_size, return_sequences=True,
input_shape=(self.time_length,
self.embedding_size),
init=self.init_type,
activation=self.activation)
stack_model = LSTM(self.hidden_size, return_sequences=True,
input_shape=(self.time_length,
self.hidden_size),
init=self.init_type,
activation=self.activation)
elif self.arch == '2rnn':
basic_model = SimpleRNN(self.hidden_size, return_sequences=True,
input_shape=(
self.time_length, self.embedding_size),
init=self.init_type,
activation=self.activation)
stack_model = SimpleRNN(self.hidden_size, return_sequences=True,
input_shape=(
self.time_length, self.hidden_size),
init=self.init_type,
activation=self.activation)
else:
basic_model = GRU(self.hidden_size, return_sequences=True,
input_shape=(self.time_length,
self.embedding_size),
init=self.init_type,
activation=self.activation)
stack_model = GRU(self.hidden_size, return_sequences=True,
input_shape=(self.time_length,
self.hidden_size),
init=self.init_type,
activation=self.activation)
model.add(basic_model)
if self.dropout:
model.add(Dropout(self.dropout_ratio))
model.add(stack_model)
model.add(TimeDistributed(Dense(self.output_vocab_size)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=opt_func)
self.model = model
else:
sys.stderr.write("Invalid arch.\n")
exit()
# save model descriptions
self.model.summary()
# 隐藏层cell的个数
cell_size = 50
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train / 255.0
x_test = x_test / 255.0
# one hot
y_train = np_utils.to_categorical(y_train, num_classes=10)
y_test = np_utils.to_categorical(y_test, num_classes=10)
model = Sequential()
model.add(SimpleRNN(
units=cell_size,
input_shape=(time_step, input_size),
))
model.add(Dense(10, activation='softmax'))
adam = Adam(lr=1e-4)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=64, epochs=10)
loss, accuracy = model.evaluate(x_test, y_test)
print(loss, " ", accuracy)
# ----------------------
# prepare dataset:
# ----------------------
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
tokenizer.fit_on_texts(dp.fetch_dataset_train().data)
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_matrix(dp.fetch_dataset_train().data,
mode='tfidf')
X_test = tokenizer.texts_to_matrix(dp.fetch_dataset_test().data, mode='tfidf')
y_train = dp.fetch_dataset_train().target
y_train = to_categorical(np.asarray(y_train))
y_test = dp.fetch_dataset_test().target
y_test = to_categorical(np.asarray(y_test))
model = Sequential()
model.add(Embedding(MAX_NUM_WORDS, 50, input_length=MAX_NUM_WORDS))
model.add(Dropout(0.25))
model.add(SimpleRNN(50, return_sequences=True))
model.add(Flatten())
model.add((Dense(NUM_CATEGORIES, activation='softmax')))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
model.fit(X_train, y_train, epochs=10, batch_size=128)
loss, accuracy = model.evaluate(X_test, y_test, batch_size=128)
print('Accuracy 3: %f' % (accuracy * 100))
# Pad Sequences
x_train = sequence.pad_sequences(x_train_seq, maxlen=500)
# Data conversion of Label
y_train = np.array(df['Label'])
# Model training
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import SimpleRNN
model_RNN = Sequential()
model_RNN.add(Embedding(input_dim=2000, input_length=500, output_dim=32))
model_RNN.add(Dropout(0.35))
model_RNN.add(SimpleRNN(units=16)) # 16層RNN
model_RNN.add(Dense(units=256,activation='relu')) #影藏層256層
model_RNN.add(Dropout(0.35))
model_RNN.add(Dense(units=1, activation='sigmoid'))
model_RNN.summary()
model_RNN.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
train_history = model_RNN.fit(x_train, y_train, batch_size=100, epochs=10, verbose=2, validation_split=0.2)
fontsize=20
def show_train_history(train_history, train, validation):
plt.plot(train_history.history[train])
plt.plot(train_history.history[validation])
plt.title('Train History', fontsize=fontsize)
plt.ylabel(train, fontsize=fontsize)
token = Tokenizer(num_words=4000)
token.fit_on_texts(train_x)
token.word_index
x_train_seq = token.texts_to_sequences(train_x)
x_test_seq = token.texts_to_sequences(test_x)
train_fit = sequence.pad_sequences(x_train_seq, maxlen=400)
test_fit = sequence.pad_sequences(x_test_seq, maxlen=400)
#2)build model
#RNN
modelRNN = Sequential()
modelRNN.add(Embedding(output_dim=32,input_dim=4000,
input_length=400))
modelRNN.add(Dropout(0.2))
modelRNN.add(SimpleRNN(units=16)) #16個神經元的RNN層
modelRNN.add(Dense(units=128,activation='relu')) #128個神經元的隱藏層
modelRNN.add(Dropout(0.3))
modelRNN.add(layers.Dense(1, activation='sigmoid')) #1個神經元的輸出層
modelRNN.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
RNN_history = modelRNN.fit(train_fit,train_y, epochs=20,
batch_size=1000, verbose=2, validation_split=0.2)
#LSTM
modelLSTM = Sequential()
modelLSTM.add(Embedding(output_dim=32,input_dim=4000,
input_length=400))
modelLSTM.add(Dropout(0.2))
modelLSTM.add(LSTM(16)) #16個神經元的LSTM層
modelLSTM.add(Dense(units=128,activation='relu')) #128個神經元的隱藏層
modelLSTM.add(Dropout(0.3))
from keras.preprocessing import sequence
x_train = sequence.pad_sequences(x_train, maxlen=200)
x_test = sequence.pad_sequences(x_test, maxlen=200)
# set up training model RNN
from keras import Sequential
from keras.layers.core import Dense, Dropout
from keras.layers import Embedding
from keras.layers.recurrent import SimpleRNN
model = Sequential()
# Embedding layer
# input 200+ training examples, and each length is 200
model.add(Embedding(output_dim=32, input_dim=12000, input_length=200))
model.add(Dropout(0.25))
# RNN layer
model.add(SimpleRNN(units=32))
model.add(Dense(units=256, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(units=6, activation='softmax'))
print(model.summary())
# there are 6 results, so use categorical_crossentropy
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# start training
# train an example pre time (text length=100), and train all example 10 times
train_history = model.fit(x_train,
trainY_oneHot,
batch_size=50,
epochs=10,
from keras.models import Sequential
from keras.layers.core import Dense,Dropout,Activation
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import SimpleRNN
model=Sequential()
model.add(Embedding(output_dim=32,
input_dim=3800, # 維度 等於 詞典大小
input_length=380))
model.add(Dropout(0.35))
#2B.建立 RNN 層 (16 個神經元)
model.add(SimpleRNN(units=16))
#2C. 建立隱藏層 (256 個神經元)
model.add(Dense(units=256,activation='relu'))
model.add(Dropout(0.35))
#2D. 建立輸出層(1 個神經元)
model.add(Dense(units=1,activation='sigmoid'))
# In[16]:
#步驟 3: 查看模型的摘要
model.summary()
def main():
maxlen = 50 #過去10日間
df = get_data()
for i in range(0, len(df) - maxlen):
data.append(df['obs'].data[i:i + maxlen])
#if df['close'].data[i + maxlen] >= 0.01:
# val = 1
#elif df['close'].data[i + maxlen] <= -0.01:
# val = -1
#else:
# val = 0
#target.append( 1 if df['close'].data[i + maxlen] >= 0.01 else 0 )
#target.append(val)
target.append(df['obs'].data[i + maxlen])
#write_data(data, 'some1.csv')
#write_data(target, 'some2.csv')
X = np.array(data).reshape(len(data), maxlen, 1)
Y = np.array(target).reshape(len(data), 1)
# データ設定
N_train = int(len(data) * 0.9)
#N_validation = len(data) - N_train
N_test = len(data) - N_train
#X_train, X_validation, Y_train, Y_validation = train_test_split(X, Y, test_size=N_validation)
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=N_test,
shuffle=False)
N_train2 = int(N_train * 0.9)
N_validation = N_train - N_train2
X_train, X_validation, Y_train, Y_validation = train_test_split(
X_train, Y_train, test_size=N_validation)
'''
モデル設定
'''
n_in = len(X[0][0]) # 1
n_hidden = 30
n_out = len(Y[0]) # 1
def weight_variable(shape, name=None):
return np.random.normal(scale=.01, size=shape)
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
model = Sequential()
model.add(
SimpleRNN(n_hidden,
kernel_initializer=weight_variable,
input_shape=(maxlen, n_in)))
model.add(Dense(n_out, kernel_initializer=weight_variable))
model.add(Activation('linear'))
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
model.compile(loss='mean_squared_error',
optimizer=optimizer,
metrics=['accuracy'])
'''
モデル学習
'''
epochs = 30
batch_size = 10
hist = model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_validation, Y_validation),
callbacks=[early_stopping])
##'''
##学習の進み具合を可視化
##'''
#val_acc = hist.history['val_acc']
#val_loss = hist.history['val_loss']
#plt.rc('font', family='serif')
#fig = plt.figure()
##plt.plot(range(len(hist.epoch)), val_acc, label='acc', color='black')
#plt.plot(range(len(hist.epoch)), val_loss, label='acc', color='black')
#plt.xlabel('epochs')
#plt.show()
#'''
#予測精度の評価
#'''
#loss_and_metrics = model.evaluate(X_validation, Y_validation)
#print('Test loss: %s, Test acc: %s' % (loss, acc))
#y_ = model.predict(X_validation[:1])
#print('---------')
#print(X_validation[:1])
#print('---------')
#print(y_)
##predict(self, x, batch_size=32, verbose=0)
#'''
#出力を用いて予測
#'''
#truncate = maxlen
#Z = X_validation[:1] # 元データの最初の一部だけ切り出し
#original = [f[i] for i in range(maxlen)]
#predicted = [None for i in range(maxlen)]
original = []
predicted = []
for i in range(N_test):
z_ = X_test[i:i + 1]
y_ = model.predict(z_)
#sequence_ = np.concatenate(
# (z_.reshape(maxlen, n_in)[1:], y_),
# axis=0).reshape(1, maxlen, n_in)
#Z = np.append(Z, sequence_, axis=0)
original.append(Y_test[i])
predicted.append(y_.reshape(-1))
#'''
#グラフで可視化
#'''
plt.rc('font', family='serif')
plt.figure()
plt.ylim([0.5, 1.0])
#plt.plot(toy_problem(T, ampl=0), linestyle='dotted', color='#aaaaaa')
plt.plot(original, linestyle='dashed', color='red')
plt.plot(predicted, color='black')
plt.show()
# 시계열 딥러닝은 자기 자신의 과거를 독립변수로 활용을 한다. 1시간 전 데이터를 독립변수로 사용해야 한다.
X_train = []
y_train = []
for i in range(1, 17000):
X_train.append(train_scaled[i - 1:i, 0])
y_train.append(train_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1)) # 3차원화
print(X_train.shape)
# 시계열 딥러닝 (RNN) : 과거의 입력값을 네트워크에 남겨서 출력에 영향을 미치게하는 기법
rnn = Sequential()
rnn.add(SimpleRNN(activation='relu', units=6, input_shape=(1, 1)))
rnn.add(Dense(activation='linear', units=1))
print(rnn.summary()) # 모델 확인
rnn.compile(loss='mse', optimizer='adam', metrics=['mse'])
rnn.fit(X_train, y_train, batch_size=1, epochs=2) # X는 1시간전 데이터, y는 원래 데이터
inputs = sc.transform(test) # 테스트 데이터도 스케일작업
print(inputs.shape)
X_test = []
for i in range(1, 415):
X_test.append(inputs[i - 1:i, 0])
X_test = np.array(X_test)
Y = np.zeros((len(input_chars), chars_count), dtype=np.bool)
for i, input_char in enumerate(input_chars):
for j, c in enumerate(input_char):
X[i, j, char2index[c]] = 1
Y[i, char2index[label_chars[i]]] = 1
HIDDEN_SIZE = 128
BATCH_SIZE = 128
NUM_ITERATIONS = 1000
NUM_EPOCHS_PER_ITERATION = 1
NUM_PREDS_PER_EPOCH = 100
model = Sequential()
model.add(
SimpleRNN(HIDDEN_XSIZE,
return_sequences=False,
input_shape=(SEQLEN, chars_count),
unroll=True))
model.add(Dense(chars_count))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
for iteration in range(NUM_ITERATIONS):
print('Iteration : %d' % iteration)
model.fit(X, Y, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS_PER_ITERATION)
# 训练1epoch,测试一次
test_idx = np.random.randint(len(input_chars))
test_chars = input_chars[test_idx]
print('test seed is : %s' % test_chars)
print(test_chars, end='')
for i in range(NUM_PREDS_PER_EPOCH):
# 测试序列向量化
def get_rnn_model(num_timesteps,
num_input_vars,
num_output_vars,
macro_dims=10,
discount=0.8,
output_type='bool',
archtype='RNN',
hidden_layer_dims=[],
hidden_activation='tanh',
macro_activation=None,
activation_props=dict(),
optimizer='rmsprop',
regularization=0.0):
if macro_activation is None:
macro_activation = hidden_activation
print "Creating %s-valued model, archtype %s, hidden activation=%s, macro activation=%s, optimizer=%s" % \
(output_type, archtype, hidden_activation, macro_activation, optimizer)
print "num_timesteps=%d, num_input_vars=%d, num_output_vars=%d" % (
num_timesteps, num_input_vars, num_output_vars)
print "macro_dims=%d, hidden_layer_dims=%s" % (macro_dims,
str(hidden_layer_dims))
print "discount factor=%0.2f" % discount
"""
class Copy(MaskedLayer):
def get_output(self, train=False):
X = self.get_input(train)
return X
class PadZerosVector2(RepeatVector):
def get_output(self, train=False):
x = self.get_input(train)
x[:,1:,:]=0.0
return x
#stacked = K.concatenate( [K.reshape(x, (x_shape[0], x_shape[1], 1)), K.zeros( (x_shape[0], x_shape[1], self.n-1) )], axis=2 )
#print self.input_length, x_shape[0]
#print self.input_
#K.zeros( (x_shape[0], x_shape[1], self.input_length-1) )
z = x * 0.0
#x_shape = (num_vars, num_samples)
#x2 = K.concatenate( [x,] + z * (self.input_length-1) , axis=3)
stacked = K.concatenate( [K.expand_dims(x), K.expand_dims(x)], axis=2 )
return K.permute_dimensions(stacked, (0, 2, 1))
"""
OUTPUT_TYPES = ('bool', 'real')
if output_type not in OUTPUT_TYPES:
print 'output_type [%s] must be one of %s' % (output_type,
OUTPUT_TYPES)
if output_type == 'bool':
output_activation = 'sigmoid'
else:
output_activation = 'linear'
model = Sequential()
def get_activation(p_act):
act = p_act.lower()
if act == 'prelu':
from keras.layers.advanced_activations import PReLU
act = PReLU(**activation_props)
elif act == 'srelu':
from keras.layers.advanced_activations import SReLU
act = SReLU(**activation_props)
elif act == 'leakyrelu':
from keras.layers.advanced_activations import LeakyReLU
act = LeakyReLU(**activation_props)
return act
hidden_activation = get_activation(hidden_activation)
macro_activation = get_activation(macro_activation)
init_dist = 'lecun_uniform'
init_dist = 'he_normal'
regobj = None
if regularization:
from keras.regularizers import l1
regobj = l1(l=regularization)
if archtype == 'rnn':
if hidden_layer_dims:
raise Exception('hidden_layer_dims not supported')
model.add(
SimpleRNN(macro_dims,
input_dim=num_input_vars,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist,
U_regularizer=regobj))
model.add(
TimeDistributed(
Dense(num_output_vars,
activation=output_activation,
init=init_dist)))
"""
elif archtype == 'rnn_identity':
if hidden_layer_dims:
raise Exception('hidden layer dis should be set to None for %s' % archtype)
if macro_dims != num_input_vars:
print "Setting macro_dims=num_input_vars for rnn_identity"
macro_dims = num_input_vars
# raise Exception('macro_dims should equal num_input_vars for %s'% archtype)
model.add(TimeDistributed(Dense(num_input_vars, input_dim=num_input_vars, input_length=num_timesteps,
activation='linear', init='identity', trainable=False)))
model.add(SimpleRNNSingleInput(num_input_vars, input_dim=num_input_vars, return_sequences=True,
input_length=num_timesteps,
activation=macro_activation, init=init_dist))
model.add(TimeDistributed(Dense(num_input_vars, input_dim=num_input_vars, input_length=num_timesteps,
activation='linear', init='identity', trainable=False)))
elif archtype == 'rnn_init_time':
c_dim = num_input_vars
for d in hidden_layer_dims:
model.add(TimeDistributed(Dense(d, input_dim=c_dim, input_length=num_timesteps,
activation=hidden_activation, init=init_dist)))
c_dim = d
model.add(SimpleRNNSingleInput(macro_dims, input_dim=c_dim, return_sequences=True,
input_length=num_timesteps,
activation=macro_activation, init=init_dist))
c_dim = macro_dims
for d in hidden_layer_dims[::-1]:
model.add(TimeDistributed(Dense(d, input_dim=c_dim, input_length=num_timesteps,
activation=hidden_activation, init=init_dist)))
c_input_dim = d
model.add(TimeDistributed(Dense(num_output_vars, input_dim=c_dim,
activation=output_activation, init=init_dist)))
"""
elif archtype == 'rnn1':
model.add(
TimeDistributed(
Dense(macro_dims,
input_dim=num_input_vars,
input_length=num_timesteps,
activation='linear',
init=init_dist)))
model.add(
SimpleRNNSingleInput2(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn2':
model.add(
TDD(
Dense(macro_dims,
input_dim=num_input_vars,
input_length=num_timesteps,
activation='linear',
init=init_dist)))
model.add(
SimpleRNNTDD(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn2_stacked':
model.add(
TDD(Dense(100,
input_dim=num_input_vars,
activation='tanh',
init=init_dist),
input_length=num_timesteps))
model.add(
TDD(
Dense(50,
input_dim=100,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
TDD(
Dense(macro_dims,
input_dim=50,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
SimpleRNNTDD(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn2_invstacked':
model.add(
TDD(
Dense(macro_dims,
input_dim=num_input_vars,
input_length=num_timesteps,
activation='linear',
init=init_dist)))
model.add(
SimpleRNNTDD(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(50,
input_dim=macro_dims,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
TimeDistributed(
Dense(100,
input_dim=50,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=100,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn3':
model.add(
TDD(Dense(macro_dims,
input_dim=num_input_vars,
activation='tanh',
init=init_dist),
input_length=num_timesteps))
model.add(
SimpleRNNTDD(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn4':
model.add(
TDD(
Dense(macro_dims,
input_dim=num_input_vars,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
SimpleRNN(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
elif archtype == 'rnn4_stacked':
model.add(
TimeDistributed(
Dense(1000,
input_dim=num_input_vars,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
TDD(
Dense(macro_dims,
input_dim=1000,
input_length=num_timesteps,
activation='tanh',
init=init_dist)))
model.add(
SimpleRNN(macro_dims,
U_regularizer=regobj,
input_dim=macro_dims,
return_sequences=True,
input_length=num_timesteps,
activation=macro_activation,
init=init_dist))
model.add(
TimeDistributed(
Dense(num_output_vars,
input_dim=macro_dims,
activation=output_activation,
init=init_dist)))
else:
raise Exception('Unknown RNN architecture %s' % archtype)
"""
elif False:
model.add(Dense(num_vars, input_dim=num_vars, activation='tanh')) # , init='uniform', activation='sigmoid'))
model.add(RepeatVector(num_timesteps))
elif False:
#activation = 'relu'
activation = 'tanh'
#model.add(Copy(input_dim=num_vars))
model.add(PadZerosVector2(num_timesteps, input_shape=(num_vars,num_timesteps)))
model.add(SimpleRNN(hidden_dims, return_sequences=True, input_length=num_timesteps, activation=activation))
model.add(TimeDistributed(Dense(num_vars, activation=output_activation)))
#model.add(Activation('sigmoid'))
elif False:
model.add(Dense(hidden_dims, input_dim=num_vars, activation='tanh')) # , init='uniform', activation='sigmoid'))
model.add(PadZerosVector(num_timesteps))
model.add(SimpleRNN(hidden_dims, return_sequences=True, input_length=num_timesteps, activation='tanh'))
model.add(TimeDistributed(Dense(num_vars, activation=output_activation)))
#model.add(Activation('sigmoid'))
elif False:
#activation = 'relu'
activation = 'tanh'
model.add(TimeDistributed(Dense(hidden_dims, input_dim=num_vars, input_length=num_timesteps, activation=activation)))
model.add(SimpleRNN(hidden_dims, return_sequences=True, input_length=num_timesteps, activation=activation))
model.add(TimeDistributed(Dense(num_vars, activation=output_activation)))
#model.add(Activation('sigmoid'))
else:
"""
if output_type == 'bool':
loss_func = binary_crossentropy
else:
loss_func = lambda y_true, y_pred: (y_true - y_pred
)**2.0 # mean_squared_error
if True:
timeweights = discount**np.arange(num_timesteps)
def get_timeweighted_loss(timeweights):
def f(y_true, y_pred):
ce = loss_func(y_pred, y_true)
ce = timeweights * ce
return K.mean(ce, axis=-1)
return f
loss = get_timeweighted_loss(timeweights[None, :, None])
#loss = 'binary_crossentropy'
print "Just doing MSE error, not timediscounted!"
model.compile(loss='mse', optimizer=optimizer)
return model
# 接下來建立模型
# 首先建立Sequential模型，之後只需要使用model.add()方法，將各個神經網路層加入模型即可
model = Sequential()
# 接下來要加入"Embedding 層"到模型中
# keras提供"Embedding 層" 可以將 "數字list" 轉成 "向量list"
# 這是word embedding的技術。剛剛將一個英文字轉成對應到的一個數字，現在將這個數字轉成一個向量
model.add(Embedding(output_dim=32, input_dim=3800, input_length=380))
# ouput_dim=32代表我們希望將"數字list"轉換成32維的向量
# imput_dim=3800 因為我們之前建立的字典含有3800個字
# input_length=380 因為每一個"數字list"中有380個數字
model.add(Dropout(0.2)) # 加入dropout避免overfitting，每次在訓練迭代時，都會隨機放棄20%的神經元
model.add(SimpleRNN(units=16)) # 建立"RNN 層"，有16個神經元
model.add(Dense(units=256, activation='relu',
kernel_initializer='normal')) # 將"隱藏層"加到模型中
model.add(Dropout(0.35)) # 加入dropout避免overfitting，每次在訓練迭代時，都會隨機放棄35%的神經元
model.add(Dense(
units=1, activation='sigmoid',
kernel_initializer='normal')) # 將"輸出層"加到模型中。輸出層只有一個神經元，1代表正面評價、0代表負面評價
print(model.summary()) # 印出該模型的摘要
# 接下來要訓練此模型
# 我們使用compile方法對訓練模習進行設定
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# 設定loss function為cross entropy error (因為只有一個output神經元，所以採用binary_crossentropy)
for i in range(0, length_of_sequences - maxlen + 1):
data.append(f[i:i + maxlen])
target.append(f[i + maxlen])
#print(data)
#print(target)
X = np.array(data).reshape(len(data), maxlen, 1)
Y = np.array(target).reshape(len(target), 1)
N_train = int(len(data) * 0.9)
N_validation = len(data) - N_train
X_train, X_validation, Y_train, Y_validation = train_test_split(X, Y, test_size=N_validation)
# 入力層
n_in = 1
# 隠れ層
n_hidden = 3
# 出力層
n_out = 1
model = Sequential()
model.add(SimpleRNN(n_hidden,
kernel_initializer=weight_variable,
input_shape=(maxlen, n_in)))
model.add(Dens(n_out, kernel_initializer=weight_varible))
model.add(Activation('linear'))
col += 1
else:
if col == 0:
y_train[count][col] = data
col += 1
else:
X_train[count - 1][col - 1][0] = data
col += 1
count += 1
print "finish data"
nb_classes = 1
hidden_units = 100
model = Sequential()
model.add(SimpleRNN(70, activation='relu', input_shape=X_train.shape[1:]))
model.add(Dense(nb_classes))
model.add(Activation('linear'))
# model.add(LSTM(5, 300, return_sequences=True))
# model.add(LSTM(300, 500, return_sequences=True))
# model.add(Dropout(0.2))
# model.add(LSTM(500, 200, return_sequences=False))
# model.add(Dropout(0.2))
# model.add(Dense(200, 3))
model.compile(loss="mean_squared_error", optimizer="rmsprop")
model.fit(X_train, y_train, nb_epoch=200, validation_split=0.15, verbose=0)
csv_data = model.predict(X_test)
test_size = int(dataX.shape[0] * 0.1)
train_size = dataX.shape[0] - test_size
X_train = dataX[0:train_size, ]
X_test = dataX[train_size:dataX.shape[0], ]
Y_train = dataY[0:train_size, ]
Y_test = dataY[train_size:dataX.shape[0], ]
X_train = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1]))
X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))
Y_train = np.reshape(Y_train, (Y_train.shape[0], 1))
Y_test = np.reshape(Y_test, (Y_test.shape[0], 1))
# 3、model train
# 1）第一次训练需要训练模型并保存模型
model = Sequential()
model.add(SimpleRNN(64))
model.add(Dropout(0.5)) # dropout层防止过拟合
model.add(Dense(1)) # 全连接层
model.add(Activation('sigmoid')) #激活层
model.compile(optimizer=RMSprop(), loss='mse')
model.fit(X_train, Y_train, nb_epoch=50, batch_size=4, verbose=2)
model.save('weather.h5')
# 2）若模型已经训练好，可以直接加载模型
# model = load_model('weather.h5')
# 4、model predict
Y_predict = model.predict(X_test)
# 5、model evaluation
Y_predict = scaler.inverse_transform(Y_predict)
np.reshape(Y_test, (Y_test.shape[0], 1))
def main():
##2次関数
#x = np.arange(-5,5,0.01)
#y = x**2
#plt.plot(y)
##plt.plot(x, y)
#plt.show()
##ノイズ三角関数
#T=200
#f = toy_problem(T)
#print(f)
#plt.plot(f)
#plt.show()
#'''
#データの生成
#'''
T = 100
T = 5
f = toy_problem(T)
length_of_sequences = 2 * T
maxlen = 25 # ひとつの時系列データの長さ
maxlen = 4
data = []
target = []
print(f)
#print(f[0:2])
#print(f[2])
for i in range(0, length_of_sequences - maxlen + 1):
data.append(f[i:i + maxlen])
target.append(f[i + maxlen])
print(data)
print('--------')
print(target)
X = np.array(data).reshape(len(data), maxlen, 1)
#print(X)
Y = np.array(target).reshape(len(data), 1)
#続き・・
# データ設定
N_train = int(len(data) * 0.9)
N_validation = len(data) - N_train
X_train, X_validation, Y_train, Y_validation = \
train_test_split(X, Y, test_size=N_validation)
'''
モデル設定
'''
n_in = len(X[0][0]) # 1
n_hidden = 20
n_out = len(Y[0]) # 1
def weight_variable(shape, name=None):
return np.random.normal(scale=.01, size=shape)
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1)
model = Sequential()
model.add(
SimpleRNN(n_hidden,
kernel_initializer=weight_variable,
input_shape=(maxlen, n_in)))
model.add(Dense(n_out, kernel_initializer=weight_variable))
model.add(Activation('linear'))
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
model.compile(loss='mean_squared_error', optimizer=optimizer)
#All you need is love
word_to_int={'All':4,'you':2,'need':5,'is':1,'love':3}
input_words=[['All','you'],['you','need'],['need','is']]
label_word=[['need'],['is'],['love']]
encoded_word=[[4,2],[2,5],[5,1]]
encoded_labels=[[5],[1],[3]]
one_hot_encoded_labels= to_categorical(encoded_labels)
print(one_hot_encoded_labels)
#(batch_size,time_steps,features)
#(3,2,1)
input_words=np.reshape(encoded_word,(3,2,1))
print(input_words.shape)
model=Sequential()
model.add(SimpleRNN(1,activation='tanh',return_sequences=False,recurrent_initializer='Zeros',input_shape=(2,1)))
model.add(Dense(6,activation='softmax'))
model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['acc'])
model.summary()
model.fit(input_words,one_hot_encoded_labels,epochs=500)
#save network
# serialize model to JSON
model_json = model.to_json()
with open("UsimpleRNN_model.json", "w") as json_file:
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
print('Evaluate IRNN...')
model = Sequential()
model.add(
SimpleRNN(output_dim=hidden_units,
init=lambda shape: normal(shape, scale=0.001),
inner_init=lambda shape: identity(shape, scale=1.0),
activation='relu',
input_shape=X_train.shape[1:]))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
rmsprop = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy', optimizer=rmsprop)
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epochs,
show_accuracy=True,
verbose=1,
validation_data=(X_test, Y_test))
n_out = image_size*image_size
def weight_variable(shape,name=None):
return np.random.normal(scale=.01,size=shape)
early_stopping = EarlyStopping(monitor='val_loss',patience=10,verbose=1)
model = Sequential()
model.add(Conv2D(t_length,image_size,use_bias = False,input_shape = (image_size,image_size,1),padding = 'valid',kernel_regularizer = binary_regularizer))
model.add(Reshape((t_length//input_dim,input_dim)))
model.add(SimpleRNN(n_hidden,kernel_initializer=weight_variable))
model.add(Dense(n_out,kernel_initializer=weight_variable))
model.add(Activation('relu'))
model.add(Reshape((image_size,image_size,1)))
model.add(Conv2D(64,9,padding = 'same'))
model.add(Activation('relu'))
model.add(Conv2D(32,1,padding = 'same'))
model.add(Activation('relu'))
model.add(Conv2D(1,5,padding = 'same'))
model.add(SimpleRNN(units=16,
return_sequences=True,
input_shape=[8, 2],
go_backwards=False,
activation='relu',
# dropout=0.5,
# recurrent_dropout=0.3,
# unroll = True,
))
'''
model.add(
SimpleRNN(
units=128,
return_sequences=True,
input_shape=[8, 2],
go_backwards=False,
activation='relu',
dropout=0.5,
recurrent_dropout=0.3,
# unroll = True,
))
# 出力層
model.add(Dense(1, activation='sigmoid', input_shape=(-1, 2)))
model.summary()
# - 最適化方法をadamに変更
# model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.1), metrics=['accuracy'])
model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
history = model.fit(x_bin, d_bin.reshape(-1, 8, 1), epochs=5, batch_size=2)
# テスト結果出力
score = model.evaluate(x_bin_test, d_bin_test.reshape(-1, 8, 1), verbose=0)
