print(vocab_size)
print(story_maxlen)
print(query_maxlen)
# 単語をインデックスに変換
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
inputs_train, queries_train, answers_train = vectorize_stories(
train_stories, word_idx, story_maxlen, query_maxlen)
inputs_test, queries_test, answers_test = vectorize_stories(
test_stories, word_idx, story_maxlen, query_maxlen)
print(inputs_train[0])
print(queries_train[0])
print(answers_train[0])
# モデルを構築
input_encoder = Sequential()
input_encoder.add(Embedding(input_dim=vocab_size, output_dim=64))
input_encoder.add(LSTM(64, return_sequences=False))
question_encoder = Sequential()
question_encoder.add(Embedding(input_dim=vocab_size, output_dim=64))
question_encoder.add(LSTM(64, return_sequences=False))
model = Sequential()
model.add(Merge([
input_encoder,
question_encoder,
]))
maxlen = 100 # cut texts after this number of words (among top max_features most common words)
batch_size = 32
print("Loading data...")
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features, test_split=0.2)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen))
model.add(LSTM(128)) # try using a GRU instead, for fun
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy', optimizer='adam', class_mode="binary")
print("Train...")
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=4, validation_data=(X_test, y_test), show_accuracy=True)
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size, show_accuracy=True)
print('Test score:', score)
print('Test accuracy:', acc)
def test_model(sequence_length, window_size, X_train_shape, dropout_value,
activation_function, loss_function, optimizer, weights, X_test,
Y_test, unnormalized_bases):
"""
Test the model on the testing data
Arguments:
model -- The previously fitted 3 layer Recurrent Neural Network
X_test -- A tensor that represents the x values of the testing data
Y_test -- A tensor that represents the y values of the testing data
unnormalized_bases -- A tensor that can be used to get unnormalized data points
Returns:
y_predict -- A tensor that represnts the normalized values that the model predicts based on X_test
real_y_test -- A tensor that represents the actual prices of XRP throughout the testing period
real_y_predict -- A tensor that represents the model's predicted prices of XRP
fig -- A branch of the graph of the real predicted prices of XRP versus the real prices of XRP
model = Sequential()
model.add(LSTM(units=sequence_length, activation='tanh', input_shape = (window_size, X_train_shape), return_sequences=False))
model.add(Dropout(0.8))
model.add(Dense(1, activation='sigmoid'))
model.add(LeakyReLU())
"""
#Create a Sequential model using Keras
model = Sequential()
#First recurrent layer with dropout
model.add(
Bidirectional(
LSTM(window_size, return_sequences=True),
input_shape=(window_size, X_train_shape),
))
model.add(Activation("relu"))
model.add(Dropout(dropout_value))
#Second recurrent layer with dropout
model.add(Bidirectional(LSTM((window_size * 2), return_sequences=True)))
model.add(Dropout(dropout_value))
#Third recurrent layer with dropout
model.add(Bidirectional(LSTM(window_size, return_sequences=False)))
model.add(Dropout(dropout_value))
#Output layer (returns the predicted value)
model.add(Dense(units=1))
#Set activation function
model.add(Activation(activation_function))
#Set loss function and optimizer
model.load_weights(weights)
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=['accuracy'])
#Test the model on X_Test
y_predict = model.predict(X_test)
#Create empty 2D arrays to store unnormalized values
real_y_test = np.zeros_like(Y_test)
real_y_predict = np.zeros_like(y_predict)
#Fill the 2D arrays with the real value and the predicted value by reversing the normalization process
for i in range(Y_test.shape[0]):
y = Y_test[i]
predict = y_predict[i]
real_y_test[i] = (y + 1) * unnormalized_bases[i]
real_y_predict[i] = (predict + 1) * unnormalized_bases[i]
#Plot of the predicted prices versus the real prices
fig = plt.figure(figsize=(10, 5))
ax = fig.add_subplot(111)
ax.set_title("XRP Price Over Time")
plt.plot(real_y_predict, color='green', label='Predicted Price')
plt.plot(real_y_test, color='red', label='Real Price')
ax.set_ylabel("Price (USD)")
ax.set_xlabel("Time (Hours)")
ax.legend()
plt.savefig("XRP-Predicted-Real-graph.png")
return y_predict, real_y_test, real_y_predict, fig
def main():
all_labels, all_text = read_files(all_dataPath)
print(len(all_labels))
###################
# Step3: Tokenize
####################
MAX_SEQUENCE_LENGTH = 300 # 最大長度
EMBEDDING_DIM = 200
VALIDATION_SPLIT = 0.16 # 驗證比例
TEST_SPLIT = 0.2 # 測試比例
tokenizer = Tokenizer()
tokenizer.fit_on_texts(all_text)
sequences = tokenizer.texts_to_sequences(all_text)
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
print(data)
labels = keras.utils.to_categorical(np.asarray(all_labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)
p1 = int(len(data) * (1 - VALIDATION_SPLIT - TEST_SPLIT))
p2 = int(len(data) * (1 - TEST_SPLIT))
train_data = data[:p1]
train_label = labels[:p1]
test_data = data[p2:]
labels_test = labels[p2:]
print('train docs: ' + str(len(train_data)))
#print ('val docs: ' + str(len(x_val)))
print('test docs: ' + str(len(test_data)))
print('test label: ' + str(len(labels_test)))
max_words = 300
batch_size = 100
epochs = 5
###################
# Step4: Building MODEL
####################
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.embeddings import Embedding
from keras.models import model_from_json
from keras.layers.recurrent import SimpleRNN, LSTM
from keras.layers import Conv1D, MaxPooling1D, Embedding
model = Sequential()
model.add(
Embedding(len(word_index) + 1,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH))
model.add(LSTM(300, dropout=0.1, recurrent_dropout=0.1))
model.add(Dropout(0.2))
model.add(Dense(labels.shape[1], activation='sigmoid'))
model.summary()
###################
# Step5: Training
###################
logger.info('Start training process...')
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(train_data,
train_label,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_split=0.3)
###################
# Step6: Evaluation
###################
logger.info('Start evaluation...')
scores = model.evaluate(test_data, labels_test, verbose=1)
print("")
logger.info('Score={}'.format(scores[1]))
def lcrn_model(input_shape, num_classes):
"""Build a CNN into RNN.
Starting version from:
https://github.com/udacity/self-driving-car/blob/master/
steering-models/community-models/chauffeur/models.py
Heavily influenced by VGG-16:
https://arxiv.org/abs/1409.1556
Also known as an LRCN:
https://arxiv.org/pdf/1411.4389.pdf
"""
model = Sequential()
model.add(
TimeDistributed(Conv2D(32, (7, 7),
strides=(2, 2),
activation='relu',
padding='same'),
input_shape=(80, 80, 3)))
model.add(
TimeDistributed(
Conv2D(32, (3, 3),
kernel_initializer="he_normal",
activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
model.add(
TimeDistributed(Conv2D(64, (3, 3), padding='same', activation='relu')))
model.add(
TimeDistributed(Conv2D(64, (3, 3), padding='same', activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
model.add(
TimeDistributed(Conv2D(128, (3, 3), padding='same',
activation='relu')))
model.add(
TimeDistributed(Conv2D(128, (3, 3), padding='same',
activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
model.add(
TimeDistributed(Conv2D(256, (3, 3), padding='same',
activation='relu')))
model.add(
TimeDistributed(Conv2D(256, (3, 3), padding='same',
activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
model.add(
TimeDistributed(Conv2D(512, (3, 3), padding='same',
activation='relu')))
model.add(
TimeDistributed(Conv2D(512, (3, 3), padding='same',
activation='relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(2, 2))))
model.add(TimeDistributed(Flatten()))
model.add(Dropout(0.5))
model.add(LSTM(256, return_sequences=False, dropout=0.5))
model.add(Dense(num_classes, activation='softmax'))
return model
X_train = X_sartrain + X_Nsartrain
X_test = X_sartest + X_Nsartest
y_train = y_sartrain + y_Nsartrain
y_test = y_sartest + y_Nsartest
#from sklearn.utils import shuffle
#X_train, y_train = shuffle(X_train, y_train)
max_length = 30
X_train = sequence.pad_sequences(X_train, maxlen=max_length)
X_test = sequence.pad_sequences(X_test, maxlen=max_length)
model = Sequential()
model.add(Embedding(80, 50, input_length=30))
model.add(LSTM(20, name="LSTM"))
model.add(Dense(1, activation='sigmoid'))
import keras.backend as K
def f1_score(y_true, y_pred):
# Count positive samples.
c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
c2 = K.sum(K.round(K.clip(y_pred, 0, 1)))
c3 = K.sum(K.round(K.clip(y_true, 0, 1)))
# If there are no true samples, fix the F1 score at 0.
if c3 == 0:
return 0
# How many selected items are relevant?
precision = c1 / c2
mask_inputs = MaskLayer(name='mask_x')(embedding_inputs)
#mask_inputs_model = Model(input=[embedding_inputs],output=mask_inputs)
mask_repeat = RepeatVector(embedding_size, name='mask_repeat')(mask_inputs)
mask_permute = Permute((2, 1), name='mask_permute')(mask_repeat)
mask_model = Model(input=[embedding_inputs], output=mask_permute)
mask_model.compile(loss='mse', optimizer=optimizer)
#Encoder Model
encoder_input = Input(shape=(maxlend, embedding_size), name='encoder_input')
encoder_mask = Masking(name='encoder_mask')(encoder_input)
encoder_layer1 = LSTM(
rnn_size,
return_sequences=True, # batch_norm=batch_norm,
W_regularizer=regularizer,
U_regularizer=regularizer,
consume_less='mem',
b_regularizer=regularizer,
dropout_W=p_W,
dropout_U=p_U,
name='encoder_layer1',
trainable=True)(encoder_mask)
encoder_layer2 = LSTM(
rnn_size,
return_sequences=True, # batch_norm=batch_norm,
W_regularizer=regularizer,
U_regularizer=regularizer,
consume_less='mem',
b_regularizer=regularizer,
dropout_W=p_W,
dropout_U=p_U,
name='encoder_layer2',
train_q1 = train_q1.apply(pd.to_numeric)
train_q2 = train_q2.apply(pd.to_numeric)
test_q1 = test_q1.apply(pd.to_numeric)
test_q2 = test_q2.apply(pd.to_numeric)
y_train = x_train['is_duplicate'].apply(pd.to_numeric).values
model = Sequential()
print('Build model...')
model5 = Sequential()
model5.add(Embedding(120000 + 1, 300, input_length=300, dropout=0.2))
model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
model6 = Sequential()
model6.add(Embedding(120000 + 1, 300, input_length=300, dropout=0.2))
model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
merged_model = Sequential()
merged_model.add(Merge([model5, model6], mode='concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
#merged_model.add(Dense(300))
# X_train = np.expand_dims(X_train,axis=(2))
y_train = np.array(trainmat['tr'][0][0][1]).squeeze()
# y_train = y_train.reshape((-1, 1))
# y_train.squeeze()
print X_train.shape
print y_train.shape
# In[3]:
lr = 1e-6 #learning rate
reg = 1e-3
print 'building model'
nb_filters = 32
model = Sequential()
model.add(
LSTM(4, W_regularizer=l2(reg), return_sequences=True, input_shape=(8, 60)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
# model.add(LSTM(64, W_regularizer=l2(reg),return_sequences=True)) # return sequences is needed for stacking
# model.add(Activation('relu'))
# model.add(Dropout(0.2))
# model.add(LSTM(128, W_regularizer=l2(reg)))
# model.add(Activation('relu'))
# model.add(Dropout(0.2))
# model.add(Dense(30))
# model.add(Flatten( input_shape=(X_train.shape[1], X_train.shape[2], X_train.shape[3])))
# model.add(Dense(10))
# model.add(Dense(1))
# model.add(LSTM(32,24, W_regularizer=l2(reg),input_shape=(X_train.shape[0], X_train.shape[1], X_train.shape[2]))) # adding conv layer collapses output
def build_model(emb_matrix, max_sequence_length):
############# Embedding Process ############
# The embedding layer containing the word vectors
emb_layer = Embedding(input_dim=emb_matrix.shape[0],
output_dim=emb_matrix.shape[1],
weights=[emb_matrix],
input_length=max_sequence_length,
trainable=False)
# Enhanced attention model ##########
# Define inputs
seq1 = Input(shape=(max_sequence_length, ))
seq2 = Input(shape=(max_sequence_length, ))
# Run inputs through embedding
emb1 = emb_layer(seq1)
emb2 = emb_layer(seq2)
# The encoder layer
# encode words with its surrouding context
encode_layer = Bidirectional(LSTM(units=300, return_sequences=True))
emb1 = Dropout(0.4)(encode_layer(emb1))
emb2 = Dropout(0.4)(encode_layer(emb2))
# score each words and calculate score matrix
cross = Dot(axes=(2, 2))([emb1, emb2])
c1 = Lambda(lambda x: keras.activations.softmax(x))(cross)
c2 = Permute((2, 1))(cross)
c2 = Lambda(lambda x: keras.activations.softmax(x))(c2)
# normalize score matrix, encoder premesis and get alignment
seq1Align = Dot((2, 1))([c1, emb2])
seq2Align = Dot((2, 1))([c2, emb1])
mm_1 = Multiply()([emb1, seq1Align])
mm_2 = Multiply()([emb2, seq2Align])
sb_1 = Lambda(lambda x: tf.subtract(x, seq1Align))(emb1)
sb_2 = Lambda(lambda x: tf.subtract(x, seq2Align))(emb2)
seq1Align = concatenate([emb1, seq1Align, mm_1, sb_1])
seq2Align = concatenate([emb2, seq2Align, mm_2, sb_2])
seq1Align = Dropout(0.4)(seq1Align)
seq2Align = Dropout(0.4)(seq2Align)
compresser = TimeDistributed(
Dense(300,
kernel_regularizer=l2(1e-5),
bias_regularizer=l2(1e-5),
activation='relu'))
seq1Align = compresser(seq1Align)
seq2Align = compresser(seq2Align)
# biLSTM Decoder
decode_layer = Bidirectional(LSTM(units=300, return_sequences=True))
final_seq1 = Dropout(0.4)(decode_layer(seq1Align))
final_seq2 = Dropout(0.4)(decode_layer(seq2Align))
averagePooling = Lambda(lambda x: K.mean(x, axis=1))
maxPooling = Lambda(lambda x: K.max(x, axis=1))
avg_seq1 = averagePooling(final_seq1)
avg_seq2 = averagePooling(final_seq2)
max_seq1 = maxPooling(final_seq1)
max_seq2 = maxPooling(final_seq2)
merged = concatenate([avg_seq1, max_seq1, avg_seq2, max_seq2])
merged = Dropout(0.4)(merged)
merged = Dense(300,
kernel_regularizer=l2(1e-5),
bias_regularizer=l2(1e-5),
activation='relu')(merged)
merged = Dropout(0.2)(merged)
merged = BatchNormalization()(merged)
pred = Dense(1, activation='sigmoid')(merged)
# model = Model(inputs=[seq1, seq2, magic_input, distance_input], outputs=pred)
model = Model(inputs=[seq1, seq2], outputs=pred)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
print(model.summary())
return model
def modelGen(self, maxlenDescs, maxlenHeads):
activationLayer = self.activationLayer
def attentionLayer(X,
mask,
n=activationLayer,
maxlenDescs=maxlenDescs,
maxlenHeads=maxlenHeads):
desc, head = X[:, :maxlenDescs, :], X[:, maxlenDescs:, :]
head_activations, headWords = head[:, :, :n], head[:, :, n:]
desc_activations, descWords = desc[:, :, :n], desc[:, :, n:]
activation = K.batch_dot(head_activations,
desc_activations,
axes=(2, 2))
activation = activation + -1e20 * K.expand_dims(
1. - K.cast(mask[:, :maxlenDescs], 'float32'), 1)
activation = K.reshape(activation, (-1, maxlenDescs))
activationWeights = K.softmax(activation)
activationWeights = K.reshape(activationWeights,
(-1, maxlenHeads, maxlenDescs))
descAvgWord = K.batch_dot(activationWeights,
descWords,
axes=(2, 1))
return K.concatenate((descAvgWord, headWords))
model = Sequential()
model.add(
Embedding(self.vocabularySize,
self.embeddingDimension,
input_length=self.sequenceLen,
embeddings_regularizer=None,
weights=[self.embeddingMatrix],
mask_zero=True,
name='embedding_1'))
model.add(
LSTM(self.lstmUnits,
return_sequences=True,
dropout=self.pW,
recurrent_dropout=self.pU,
name='lstm_1'))
model.add(Dropout(self.pDense, name='dropout_1'))
model.add(
LSTM(self.lstmUnits,
return_sequences=True,
dropout=self.pW,
recurrent_dropout=self.pU,
name='lstm_2'))
model.add(Dropout(self.pDense, name='dropout_2'))
model.add(
LSTM(self.lstmUnits,
return_sequences=True,
dropout=self.pW,
recurrent_dropout=self.pU,
name='lstm_3'))
model.add(Dropout(self.pDense, name='dropout_3'))
model.add(
Lambda(activationLayer,
mask=lambda inputs, mask: mask[:, maxlenDescs:],
output_shape=lambda input_shape:
(input_shape[0], maxlenHeads, 2 *
(self.lstmUnits - activationLayer)),
name='attentionLayer_1'))
model.add(
TimeDistributed(
Dense(self.vocabularySize, name='timedistributed_1')))
model.add(Activation('softmax', name='activation_1'))
return model
y_train = []
for i in range(0, sample_size - max_len):
x_train.append(raw_text[i : i+max_len]) # Will give an array of strings
y_train.append(raw_text[i+max_len])
X_train = np.zeros((len(x_train), max_len, num_chars), dtype=np.bool)
Y_train = np.zeros((len(y_train), num_chars), dtype=np.bool)
for i in range(0, len(x_train)):
for j,char in enumerate(x_train[i]):
X_train[i, j, char_to_index[char]] = 1
Y_train[i, char_to_index[y_train[i]]] = 1
model = Sequential()
model.add(LSTM(512, input_shape=(max_len, num_chars), return_sequences=True))
model.add(Dropout(0.5))
model.add(LSTM(512, return_sequences=False))
model.add(Dropout(0.5))
model.add(Dense(num_chars))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer='RMSprop')
# model.fit(X_train, Y_train, batch_size=100, nb_epoch=5)
# model.save_weights('gandhi1.h5')
model.load_weights('gandhi1.h5')
# helper function to sample an index from a probability array
# Higher temperature -> More diversity but more mistakes
print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
test_split=0.2)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.5))
model.add(LSTM(128, dropout_W=0.5,
dropout_U=0.1)) # try using a GRU instead, for fun
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy', optimizer='adam')
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=15,
validation_data=(X_test, y_test),
show_accuracy=True)
score, acc = model.evaluate(X_test,
sys.stdout = open(filepath + '/out.txt', 'w+')
# model
model = Sequential()
model.add(
Embedding(input_dim=n_symbols,
output_dim=embed_size,
trainable=False,
input_length=maxlen,
weights=[embedding_weights],
mask_zero=True))
model.add(Dropout(rate=0.5))
model.add(
Bidirectional(
LSTM(hidden_units,
return_sequences=True,
kernel_initializer='random_uniform')))
model.add(Dropout(rate=0.5))
model.add(
Bidirectional(
LSTM(hidden_units,
return_sequences=True,
kernel_initializer='random_uniform')))
model.add(Dropout(rate=0.5))
model.add(
Bidirectional(
LSTM(hidden_units,
return_sequences=True,
kernel_initializer='random_uniform')))
