class LSTMSequenceEmbedder(SequenceEmbedder):
"""Forward LSTM Sequence Embedder
Also provide attention states.
"""
def __init__(self, token_embeds, seq_length, align='left', name='LSTMSequenceEmbedder', hidden_size=50):
self.hidden_size = hidden_size
super(LSTMSequenceEmbedder, self).__init__(token_embeds, align=align, seq_length=seq_length, name=name)
def embed_sequences(self, embed_sequence_batch):
self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
# Pass input through the LSTMs
# Shape: (batch_size, seq_length, hidden_size)
hidden_state_values = self._forward_lstm(embed_sequence_batch.values, embed_sequence_batch.mask)
self._hidden_states = SequenceBatch(hidden_state_values, embed_sequence_batch.mask)
# Embedding dimension: (batch_size, hidden_size)
shape = tf.shape(embed_sequence_batch.values)
forward_final = tf.slice(hidden_state_values, [0, shape[1] - 1, 0], [-1, 1, self.hidden_size])
return tf.squeeze(forward_final, [1])
@property
def weights(self):
return self._forward_lstm.get_weights()
@weights.setter
def weights(self, w):
self._forward_lstm.set_weights(w)
@property
def hidden_states(self):
return self._hidden_states
def embed_sequences(self, embed_sequence_batch):
"""Return sentence embeddings as a tensor with with shape
[batch_size, hidden_size * 2]
"""
forward_values = embed_sequence_batch.values
forward_mask = embed_sequence_batch.mask
backward_values = tf.reverse(forward_values, [False, True, False])
backward_mask = tf.reverse(forward_mask, [False, True])
# Initialize LSTMs
self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
self._backward_lstm = LSTM(self.hidden_size, return_sequences=True)
# Pass input through the LSTMs
# Shape: (batch_size, seq_length, hidden_size)
forward_seq = self._forward_lstm(forward_values, forward_mask)
forward_seq.set_shape((None, self.seq_length, self.hidden_size))
backward_seq = self._backward_lstm(backward_values, backward_mask)
backward_seq.set_shape((None, self.seq_length, self.hidden_size))
# Stitch the outputs together --> hidden states (for computing attention)
# Final dimension: (batch_size, seq_length, hidden_size * 2)
lstm_states = tf.concat(2, [forward_seq, tf.reverse(backward_seq, [False, True, False])])
self._hidden_states = SequenceBatch(lstm_states, forward_mask)
# Stitch the final outputs together --> sequence embedding
# Final dimension: (batch_size, hidden_size * 2)
seq_length = tf.shape(forward_values)[1]
forward_final = tf.slice(forward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
backward_final = tf.slice(backward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
return tf.squeeze(tf.concat(2, [forward_final, backward_final]), [1])
def train_keras_nn():
r = 1e-3
graph = Graph()
# graph.add_input(name='input_features', input_shape=(input_features.shape[1],))
graph.add_input(name='train_product_nnid', input_shape=(1,), dtype=int)
graph.add_input(name='query_indexes', input_shape=(15,), dtype=int)
graph.add_node(Embedding(input_dim=len(prod) + 1,
output_dim=100,
weights=[np.concatenate((np.zeros((1, 100)), product_vectors), axis=0)],
trainable=True,
input_length=1,
W_regularizer=l2(r)),
name='embedding',
input='train_product_nnid', )
graph.add_node(Flatten(), name='flatten', input='embedding')
graph.add_node(Embedding(input_dim=len(vocab_vectors) + 1,
output_dim=50,
weights=[np.concatenate((np.zeros((1, 50)), vocab_vectors), axis=0)],
trainable=True,
input_length=1,
W_regularizer=l2(r), mask_zero=True),
name='q_embedding',
input='query_indexes', )
lstm = LSTM(output_dim=50)
graph.add_node(lstm, name='LSTM', input='q_embedding')
graph.add_node(Dense(100, activation='sigmoid', W_regularizer=l2(r)), name='hidden0',
inputs=['LSTM', 'flatten'],
merge_mode="concat", concat_axis=1)
graph.add_node(Dense(1, activation='sigmoid', W_regularizer=l2(r)), name='output', input='hidden0',
create_output=True)
graph.compile(optimizer='adam', loss={'output': 'mse'})
select_best = SelectBestValidation(graph)
# To get weights of query vectors
get_lstm_output = theano.function([graph.inputs['query_indexes'].input], lstm.get_output(train=False))
graph.fit(
{'train_product_nnid': np.concatenate([train_product_nnid, test_product_nnid]),
'query_indexes': np.concatenate([query_train, query_test]),
'output': (output - 1) / 2},
batch_size=5000, nb_epoch=30, verbose=1, shuffle=True, callbacks=[select_best], validation_split=0.1)
return graph, get_lstm_output
def embed_sequences(self, embed_sequence_batch):
self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
# Pass input through the LSTMs
# Shape: (batch_size, seq_length, hidden_size)
hidden_state_values = self._forward_lstm(embed_sequence_batch.values, embed_sequence_batch.mask)
self._hidden_states = SequenceBatch(hidden_state_values, embed_sequence_batch.mask)
# Embedding dimension: (batch_size, hidden_size)
shape = tf.shape(embed_sequence_batch.values)
forward_final = tf.slice(hidden_state_values, [0, shape[1] - 1, 0], [-1, 1, self.hidden_size])
return tf.squeeze(forward_final, [1])
class BidiLSTMSequenceEmbedder(SequenceEmbedder):
"""Bidirectional LSTM Sequence Embedder
Also provide attention states.
"""
def __init__(self, token_embeds, seq_length, align='left', name='BidiLSTMSequenceEmbedder', hidden_size=50):
self.seq_length = seq_length
self.hidden_size = hidden_size
super(BidiLSTMSequenceEmbedder, self).__init__(token_embeds, align=align, seq_length=seq_length, name=name)
def embed_sequences(self, embed_sequence_batch):
"""Return sentence embeddings as a tensor with with shape
[batch_size, hidden_size * 2]
"""
forward_values = embed_sequence_batch.values
forward_mask = embed_sequence_batch.mask
backward_values = tf.reverse(forward_values, [False, True, False])
backward_mask = tf.reverse(forward_mask, [False, True])
# Initialize LSTMs
self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
self._backward_lstm = LSTM(self.hidden_size, return_sequences=True)
# Pass input through the LSTMs
# Shape: (batch_size, seq_length, hidden_size)
forward_seq = self._forward_lstm(forward_values, forward_mask)
forward_seq.set_shape((None, self.seq_length, self.hidden_size))
backward_seq = self._backward_lstm(backward_values, backward_mask)
backward_seq.set_shape((None, self.seq_length, self.hidden_size))
# Stitch the outputs together --> hidden states (for computing attention)
# Final dimension: (batch_size, seq_length, hidden_size * 2)
lstm_states = tf.concat(2, [forward_seq, tf.reverse(backward_seq, [False, True, False])])
self._hidden_states = SequenceBatch(lstm_states, forward_mask)
# Stitch the final outputs together --> sequence embedding
# Final dimension: (batch_size, hidden_size * 2)
seq_length = tf.shape(forward_values)[1]
forward_final = tf.slice(forward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
backward_final = tf.slice(backward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
return tf.squeeze(tf.concat(2, [forward_final, backward_final]), [1])
@property
def weights(self):
return (self._forward_lstm.get_weights(), self._backward_lstm.get_weights())
@weights.setter
def weights(self, w):
forward_weights, backward_weights = w
self._forward_lstm.set_weights(forward_weights)
self._backward_lstm.set_weights(backward_weights)
@property
def hidden_states(self):
"""Return a SequenceBatch whose value has shape
[batch_size, max_seq_length, hidden_size * 2]
"""
return self._hidden_states
seq_in = raw_text[i:i + seq_length]
seq_out = raw_text[i + seq_length]
dataX.append([char_to_int[char] for char in seq_in])
dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print "Total Patterns: ", n_patterns
# reshape X to be [samples, time steps, features]
X = numpy.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# define the LSTM model
model = Sequential()
model.add(
LSTM(512, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
#model.add(LSTM(256, return_sequences=True))
#model.add(Dropout(0.2))
#model.add(LSTM(128, return_sequences=True))
#model.add(Dropout(0.2))
model.add(LSTM(256))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
# load the network weights
#filename = "weights/weights-improvement-most-recent.hdf5"
#filename = "weights/multilayer-weights.hdf5"
filename = "weights/sherlock-4-2l-512n-1000e-128bs-02do-507.hdf5"
model.load_weights(filename)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.1,
def step(self, x, states):
wx = x[:, :self.da_dim]
da = x[:, self.da_dim:]
h, states = LSTM.step(self, wx, states)
states[1] = states[1] + self.activation(da) # c
return h, states[:2]
embedding_matrix = np.zeros((len(word_index) + 1, embed_size))
absent_words = 0
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
else:
absent_words += 1
# -----------------embedding layer---------------------------------------------------
embedding_layer = Embedding(len(word_index) + 1, embed_size, weights=[embedding_matrix], input_length=max_senten_len,
trainable=False)
word_input = Input(shape=(max_senten_len,), dtype='float32')
word_sequences = embedding_layer(word_input)
word_lstm = Bidirectional(LSTM(HAN_LSTM_UNITS, return_sequences=True, kernel_regularizer=l2_reg))(word_sequences)
word_dense = TimeDistributed(Dense(4, kernel_regularizer=l2_reg))(word_lstm)
word_att = AttentionWithContext()(word_dense)
wordEncoder = Model(word_input, word_att)
sent_input = Input(shape=(max_senten_num, max_senten_len), dtype='float32')
sent_encoder = TimeDistributed(wordEncoder)(sent_input)
sent_lstm = Bidirectional(LSTM(HAN_LSTM_UNITS, return_sequences=True, kernel_regularizer=l2_reg))(sent_encoder)
sent_dense = TimeDistributed(Dense(4, kernel_regularizer=l2_reg))(sent_lstm)
sent_att = Dropout(DROP_OUT_HAN)(AttentionWithContext()(sent_dense))
preds = Dense(1, activation='sigmoid')(sent_att)
model = Model(sent_input, preds)
lr = 0.01
optimizer = optimizers.Adam(lr)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
if args.command == 'train':
y_train.append(training_set_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dropout
from keras.layers import Dense
model = Sequential()
model.add(
LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], 1)))
model.add(Dropout(0.2))
model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=50))
model.add(Dropout(0.2))
model.add(Dense(units=1))
model.compile(optimizer='adam', loss='mean_squared_error')
#2. 모델 구성 # model = Sequential() # # model.add(LSTM(10, activation = 'relu', input_shape = (3,2))) # 실질적으로 행은 큰 영향을 안미치기 때문에 무시 몇개의 칼럼을 가지고 몇개씩 작업을 할 것인가 # model.add(LSTM(130, input_length = 3, input_dim = 1)) # model.add(Dense(149, activation= 'sigmoid')) # model.add(Dense(35)) # model.add(Dense(60)) # model.add(Dense(90)) # model.add(Dense(690)) # model.add(Dense(610)) # model.add(Dense(470)) # model.add(Dense(250)) # model.add(Dense(1)) input1 = Input(shape=(3, 1)) dense1 = LSTM(20)(input1) dense1 = Dense(420)(dense1) output1 = Dense(1)(dense1) model = Model(input=input1, output=output1) model.summary() # 실행 from keras.callbacks import EarlyStopping earlystopping = EarlyStopping(monitor='loss', patience=10, mode='min') model.compile(optimizer='adam', loss='mse') model.fit(x, y, epochs=1000, batch_size=10, callbacks=[earlystopping]) x_predict = array([50, 60, 70])
num_batches = 300
batch_size=20
batch_length=20
batch_X,batch_Y = batchify(categorized,categorizedY,num_batches,batch_size,batch_length)
#debug examples...
#char_X = [char_indices[c] for c in X]
#char_Y = [char_indices[c] for c in Y]
#batch_X,batch_Y = batchify(np.asarray([char_indices[c] for c in X]).reshape(len(X),1),np.asarray([char_indices[c] for c in Y]).reshape(len(Y),1),10,10,10)
# create and fit the LSTM network
print('Building the Model layers')
model = Sequential()
model.add(LSTM(80, return_sequences=True,input_shape=(batch_size,len(chars)),stateful=True,batch_size=batch_size))
model.add(Dropout(0.2))
model.add(TimeDistributed(Dense(len(chars))))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
char_indices = dict((c, i) for i, c in enumerate(chars))
print('Starting to learn:')
for j in range(5):
model.reset_states()
for i in range(num_batches):
print('{} out of {}, "epoch": {}'.format(i,num_batches,j))
model.train_on_batch(batch_X[i], batch_Y[i])
#test on batch for evaluation
def get_model_conv2d(input_profile_names, target_profile_names, scalar_input_names,
actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):
max_profile_lookback = 0
for sig in input_profile_names:
if lookbacks[sig] > max_profile_lookback:
max_profile_lookback = lookbacks[sig]
max_actuator_lookback = 0
for sig in actuator_names:
if lookbacks[sig] > max_actuator_lookback:
max_actuator_lookback = lookbacks[sig]
max_scalar_lookback = 0
for sig in scalar_input_names:
if lookbacks[sig] > max_scalar_lookback:
max_scalar_lookback = lookbacks[sig]
num_profiles = len(input_profile_names)
num_targets = len(target_profile_names)
num_actuators = len(actuator_names)
num_scalars = len(scalar_input_names)
if 'max_channels' in kwargs:
max_channels = kwargs['max_channels']
else:
max_channels = 32
profile_inputs = []
profiles = []
for i in range(num_profiles):
profile_inputs.append(
Input((lookbacks[input_profile_names[i]], profile_length), name='input_' + input_profile_names[i]))
profiles.append(Reshape((max_profile_lookback, profile_length, 1))
(ZeroPadding1D(padding=(max_profile_lookback - lookbacks[input_profile_names[i]], 0))(profile_inputs[i])))
profiles = Concatenate(axis=-1)(profiles)
# shape = (lookback, length, channels=num_profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/8), kernel_size=(1, int(profile_length/12)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/4), kernel_size=(1, int(profile_length/8)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/2), kernel_size=(1, int(profile_length/6)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, int(profile_length/4)),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
# shape = (lookback, length, channels)
if max_profile_lookback > 1:
profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(profile_lookback, 1),
strides=(1, 1), padding='valid', activation=std_activation)(profiles)
profiles = Reshape((profile_length, int(
num_profiles*max_channels)))(profiles)
# shape = (length, channels)
if num_scalars > 0:
scalar_inputs = []
scalars = []
for i in range(num_scalars):
scalar_inputs.append(
Input((lookbacks[scalar_input_names[i]],), name='input_' + scalar_input_names[i]))
scalars.append(Reshape((lookbacks[scalar_input_names[i]],1))(scalar_inputs[i]))
scalars[i] = ZeroPadding1D(padding=(max_scalar_lookback - lookbacks[scalar_input_names[i]], 0))(scalars[i])
scalars = Concatenate(axis=-1)(scalars)
# shaoe = (time, num_actuators)
scalars = Dense(units=int(num_profiles*max_channels/8),
activation=std_activation)(scalars)
# actuators = Conv1D(filters=int(num_profiles*max_channels/8), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
scalars = Dense(units=int(num_profiles*max_channels/4),
activation=std_activation)(scalars)
# actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
scalars = Dense(units=int(num_profiles*max_channels/2),
activation=std_activation)(scalars)
scalars = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
recurrent_activation='hard_sigmoid')(scalars)
scalars = Reshape((int(num_profiles*max_channels), 1))(scalars)
# shape = (channels, 1)
scalars = Dense(units=int(profile_length/4),
activation=std_activation)(scalars)
scalars = Dense(units=int(profile_length/2),
activation=std_activation)(scalars)
scalars = Dense(units=profile_length, activation=None)(scalars)
# shape = (channels, profile_length)
scalars = Permute(dims=(2, 1))(scalars)
# shape = (profile_length, channels)
actuator_future_inputs = []
actuator_past_inputs = []
actuators = []
for i in range(num_actuators):
actuator_future_inputs.append(
Input((lookahead, ), name='input_future_' + actuator_names[i]))
actuator_past_inputs.append(
Input((lookbacks[actuator_names[i]], ), name='input_past_' + actuator_names[i]))
actuators.append(Concatenate(
axis=1)([actuator_past_inputs[i], actuator_future_inputs[i]]))
actuators[i] = Reshape(
(lookbacks[actuator_names[i]]+lookahead, 1))(actuators[i])
actuators[i] = ZeroPadding1D(padding=(max_actuator_lookback - lookbacks[actuator_names[i]], 0))(actuators[i])
actuators = Concatenate(axis=-1)(actuators)
# shaoe = (time, num_actuators)
actuators = Dense(units=int(num_profiles*max_channels/8),
activation=std_activation)(actuators)
# actuators = Conv1D(filters=int(num_profiles*max_channels/8), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
actuators = Dense(units=int(num_profiles*max_channels/4),
activation=std_activation)(actuators)
# actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
# padding='causal', activation=std_activation)(actuators)
actuators = Dense(units=int(num_profiles*max_channels/2),
activation=std_activation)(actuators)
actuators = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
recurrent_activation='hard_sigmoid')(actuators)
actuators = Reshape((int(num_profiles*max_channels), 1))(actuators)
# shape = (channels, 1)
actuators = Dense(units=int(profile_length/4),
activation=std_activation)(actuators)
actuators = Dense(units=int(profile_length/2),
activation=std_activation)(actuators)
actuators = Dense(units=profile_length, activation=None)(actuators)
# shape = (channels, profile_length)
actuators = Permute(dims=(2, 1))(actuators)
# shape = (profile_length, channels)
if num_scalars > 0:
merged = Add()([profiles, actuators, scalars])
else:
merged = Add()([profiles, actuators])
merged = Reshape((1, profile_length, int(
num_profiles*max_channels)))(merged)
# shape = (1, length, channels)
prof_act = []
for i in range(num_targets):
prof_act.append(Conv2D(filters=max_channels, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
padding='same', activation=std_activation)(merged))
# shape = (1,length,max_channels)
prof_act[i] = Conv2D(filters=int(max_channels/2), kernel_size=(1, int(profile_length/8)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=int(max_channels/4), kernel_size=(1, int(profile_length/6)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=int(max_channels/8), kernel_size=(1, int(profile_length/4)),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=1, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
padding='same', activation=None)(prof_act[i])
# shape = (1,length,1)
if kwargs.get('predict_mean'):
prof_act[i] = GlobalAveragePooling2D()(prof_act[i])
else:
prof_act[i] = Reshape((profile_length,), name='target_' +
target_profile_names[i])(prof_act[i])
model_inputs = profile_inputs + actuator_past_inputs + actuator_future_inputs
if num_scalars > 0:
model_inputs += scalar_inputs
model_outputs = prof_act
model = Model(inputs=model_inputs, outputs=model_outputs)
return model
test_size = length - train_size
batch_size_array = [8, 16, 32, 64, 128]
trainX, trainY = data[0:train_size], CPU_nomal[0:train_size]
testX = data[train_size:length]
testY = dataset.T[1][train_size:length]
# reshape input to be [samples, time steps, features]
trainX = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
testX = np.reshape(testX, (testX.shape[0], testX.shape[1], 1))
# create and fit the LSTM network
for batch_size in batch_size_array:
print "batch_size= ", batch_size
model = Sequential()
model.add(
LSTM(512, return_sequences=True, activation='relu',
input_shape=(2, 1)))
model.add(LSTM(4))
model.add(Dense(1))
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['mean_squared_error'])
history = model.fit(trainX,
trainY,
epochs=2000,
batch_size=batch_size,
verbose=2,
validation_split=0.1,
callbacks=[
EarlyStopping(monitor='loss',
patience=20,
verbose=1), tensorboard
x_train_titles = x_train.loc[:,x_train.columns.isin(['title_a', 'title_b'])] x_train_emebeddings = x_train.loc[:,~x_train.columns.isin(['title_a', 'title_b'])] x_test_titles = x_test.loc[:,x_train.columns.isin(['title_a', 'title_b'])] x_test_emebeddings = x_test.loc[:,~x_train.columns.isin(['title_a', 'title_b'])] ''' GLOVE_EMBEDDING_DIMS = 100 a_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32') b_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32') shared_model = Sequential() shared_model.add(Embedding(num_words, GLOVE_EMBEDDING_DIMS, input_length=MAX_SEQUENCE_LENGTH, weights= [embedding_matrix], trainable=False)) shared_model.add(LSTM(100)) # Overarching model. a_features = shared_model(a_input) b_features = shared_model(b_input) siamese_model = concatenate([a_features, b_features]) siamese_model = Dropout(0.2)(siamese_model) preds = Dense(1, activation='sigmoid')(siamese_model) model = Model(inputs=[a_input, b_input], outputs=preds) model.compile(optimizer='adam', loss='mse', metrics=['mae', rmse]) """ model = Sequential() model.add(Embedding(num_words, GLOVE_EMBEDDING_DIMS, input_length=MAX_SEQUENCE_LENGTH, weights= [embedding_matrix], trainable=False)) model.add(Dropout(0.2))
next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))
print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
for t, char in enumerate(sentence):
X[i, t, char_indices[char]] = 1
y[i, char_indices[next_chars[i]]] = 1
# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
def sample(preds, temperature=1.0):
# helper function to sample an index from a probability array
preds = np.asarray(preds).astype('float64')
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
preds = exp_preds / np.sum(exp_preds)
probas = np.random.multinomial(1, preds, 1)
return np.argmax(probas)
testY = dataY(test, look_back)
print('Conjunto de entrenamientoX t: \n' + str(trainX))
print('Con shape' + str(trainX.shape))
print('Conjunto de entrenamientoY t+1: \n' + str(trainY))
print('Con shape' + str(trainY.shape))
print('Conjunto pruebaX t: \n' + str(testX))
print('Conjunto pruebaY t+1: \n' + str(testY))
#Create and fit the LSTM network
model = Sequential()
model.add(LSTM(4, input_shape=(look_back,1)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15, min_delta=0.001)
history = model.fit(trainX, trainY, epochs=20, validation_split=0.3, batch_size=1, verbose=2, callbacks=[es])
#Represent loss
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
pyplot.show()
#Make prediction
trainPredict = model.predict(trainX)
train_size = int(len(dataset) * 0.67)
test_size = len(dataset) - train_size
train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :]
# reshape into X=t and Y=t+1
look_back = 3
trainX, trainY = create_dataset(train, look_back)
testX, testY = create_dataset(test, look_back)
# reshape input to be [samples, time steps, features]
trainX = numpy.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
testX = numpy.reshape(testX, (testX.shape[0], testX.shape[1], 1))
# create and fit the LSTM network
batch_size = 1
model = Sequential()
model.add(
LSTM(4,
batch_input_shape=(batch_size, look_back, 1),
stateful=True,
return_sequences=True))
model.add(LSTM(4, batch_input_shape=(batch_size, look_back, 1), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(100):
model.fit(trainX,
trainY,
epochs=1,
batch_size=batch_size,
verbose=2,
shuffle=False)
model.reset_states()
# make predictions
trainPredict = model.predict(trainX, batch_size=batch_size)
model.reset_states()
t = pickle.load(open('data/tokenizer.p', 'rb'))
Tx = 10
Ty = 10
m = X.shape[0]
repeator = RepeatVector(Tx)
concatenator = Concatenate(axis=-1)
densor1 = Dense(10, activation="tanh")
densor2 = Dense(1, activation="relu")
activator = Activation(
softmax, name='attention_weights'
) # We are using a custom softmax(axis = 1) loaded in this notebook
dotor = Dot(axes=1)
n_a = 32
n_s = 64
post_activation_LSTM_cell = LSTM(n_s, return_state=True)
output_layer = Dense(len(word_index) + 1, activation=softmax)
model = model(Tx, Ty, n_a, n_s, len(word_index) + 1, len(word_index) + 1)
opt = Adam(lr=0.008, beta_1=0.9, beta_2=0.999, decay=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
s0 = np.zeros((m, n_s))
c0 = np.zeros((m, n_s))
outputs = list(Y.swapaxes(0, 1))
if os.path.isfile(
print(y_train.shape) print(y_train[0]) ###################### build network #################### ######## word dim 词向量维度 ######## word_dim = 8 ######## network structure ######## model = Sequential() #### Embedding层 #### model.add(Embedding(input_dim=1000, output_dim=word_dim, input_length=max_len)) #### 两层LSTM，第一层，设置return_sequences参数为True #### model.add(LSTM(256, return_sequences=True)) #### dropout #### model.add(Dropout(0.5)) #### 两层LSTM，第二层，设置return_sequences参数为False #### model.add(LSTM(256, return_sequences=False)) #### dropout #### model.add(Dropout(0.5)) #### 输出层 #### model.add(Dense(num_class, activation='softmax')) print(model.summary())
# # Модель нейронной сети
# In[22]:
graph = tf.Graph()
with graph.as_default():
input_X = tf.placeholder(tf.float32,
shape=[None, None, 13],
name="input_X")
labels = tf.sparse_placeholder(tf.int32)
seq_lens = tf.placeholder(tf.int32, shape=[None], name="seq_lens")
model = Sequential()
model.add(
Bidirectional(LSTM(128, return_sequences=True, implementation=2),
input_shape=(None, 13)))
model.add(Bidirectional(LSTM(128, return_sequences=True,
implementation=2)))
model.add(TimeDistributed(Dense(len(inv_mapping) + 2)))
final_seq_lens = seq_lens
logits = model(input_X)
logits = tf.transpose(logits, [1, 0, 2])
ctc_loss = tf.reduce_mean(tf.nn.ctc_loss(labels, logits, final_seq_lens))
# ctc_greedy_decoder? merge_repeated=True
decoded, log_prob = tf.nn.ctc_greedy_decoder(logits, final_seq_lens)
ler = tf.reduce_mean(
tf.edit_distance(tf.cast(decoded[0], tf.int32), labels))
def get_model_linear_systems(input_profile_names, target_profile_names, scalar_input_names,
actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):
profile_inshape = (profile_lookback, profile_length)
actuator_inshape = (actuator_lookback + lookahead,)
num_profiles = len(input_profile_names)
num_targets = len(target_profile_names)
num_actuators = len(actuator_names)
profile_inputs = []
for i in range(num_profiles):
profile_inputs.append(
Input(profile_inshape, name='input_' + input_profile_names[i]))
if num_profiles > 1:
profiles = Concatenate(axis=-1)(profile_inputs)
else:
profiles = profile_inputs[0]
profile_response = Dense(
int(profile_length/2*num_profiles), activation=std_activation)(profiles)
profile_response = Dense(
int(profile_length/2*num_profiles), activation=std_activation)(profile_response)
if profile_lookback > 1:
profile_response = LSTM(int(profile_length/2*num_profiles), activation=std_activation,
recurrent_activation='hard_sigmoid',
return_sequences=True)(profile_response)
else:
profile_response = Dense(int(profile_length/2*num_profiles),
activation=std_activation)(profile_response)
profile_response = Dense(int(profile_length/2*num_profiles),
activation=std_activation)(profile_response)
actuator_inputs = []
actuators = []
for i in range(num_actuators):
actuator_inputs.append(
Input(actuator_inshape, name='input_' + actuator_names[i]))
actuators.append(
Reshape((actuator_lookback+lookahead, 1))(actuator_inputs[i]))
if num_actuators > 1:
actuators = Concatenate(axis=-1)(actuators)
else:
actuators = actuators[0]
actuator_response = Dense(
profile_lookback, activation=std_activation)(actuators)
actuator_response = Dense(
profile_lookback, activation=std_activation)(actuator_response)
actuator_response = LSTM(profile_lookback, activation=std_activation,
recurrent_activation='hard_sigmoid',
return_sequences=True)(actuator_response)
total_response = Dot(axes=(2, 1))([actuator_response, profile_response])
total_response = LSTM(int(profile_length/2*num_targets), activation=std_activation,
recurrent_activation='hard_sigmoid')(total_response)
total_response = Dense(int(profile_length*.75*num_targets),
activation=std_activation)(total_response)
total_response = Dense(profile_length*num_targets)(total_response)
total_response = Reshape((num_targets*profile_length, 1))(total_response)
targets = []
for i in range(num_targets):
targets.append(Cropping1D(cropping=(i*profile_length,
(num_targets-i-1)*profile_length))(total_response))
targets[i] = Reshape((profile_length,),
name='target_' + target_profile_names[i])(targets[i])
model = Model(inputs=profile_inputs+actuator_inputs, outputs=targets)
return model
embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
model = Sequential()
model.add(embedding_layer)
model.add(LSTM(LSTM_OUT, dropout_U=0.25, dropout_W=0.25))
model.add(Dense(NUM_CLASSES, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
checkpoint = ModelCheckpoint(weights,
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
early = EarlyStopping(monitor='val_acc',
min_delta=0,
hidden_units = 100
batch_size = 2
epochs = 100
learning_rate = 0.01
x_train = data_XX
y_train = data_YY
x_test = data_XX
y_test = data_YY
print("x_train.shape[1:] : ", x_train.shape[1:])
model = Sequential()
model.add(LSTM(hidden_units,
return_sequences=True,
input_shape=x_train.shape[1:]))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
rmsprop = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
scores = model.evaluate(x_test, y_test, verbose=1)
print("number of words in vocab: ", len(vocab))
print("number of poems: ", len(poems))
print(len(seq_poems))
print(len(img_names))
print(len(features))
print("length of longest poem: ", max_poem_len)
inputs1 = Input(shape=(2048,))
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation='relu')(fe1)
inputs2 = Input(shape=(max_poem_len,))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
se3 = LSTM(256)(se2)
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
outputs = Dense(vocab_size, activation='softmax')(decoder2)
model = Model(inputs=[inputs1, inputs2], outputs=outputs)
model.layers[2].set_weights([embedding_matrix])
model.layers[2].trainable = False
model.compile(loss='categorical_crossentropy', optimizer='adam')
##############################
epochs = 2
number_pics_per_batch = 3
num_photos_per_batch = len(poems) #number_pics_per_batch
lst.append(lst.pop(0))
lst.reverse()
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=15,
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
def create_CNN_LSTM_POS_model(vocabulary_size, sequence_len, embedding_matrix,
EMBEDDING_SIZE, pos_tag_list_len, len_features):
max_seq_length = sequence_len
deep_inputs = Input(shape=(max_seq_length, ))
embedding = Embedding(vocabulary_size,
EMBEDDING_SIZE,
input_length=sequence_len,
weights=[embedding_matrix],
trainable=False)(deep_inputs) # line A
pos_tagging = Input(shape=(pos_tag_list_len, 1))
other_features = Input(shape=(len_features, 1))
dense_1 = Dense(4, activation="sigmoid")(other_features)
dense_2 = Dense(8, activation="sigmoid")(dense_1)
dense_3 = Dense(16, activation="sigmoid")(dense_2)
dropout_rate = 0.5
def convolution_and_max_pooling(input_layer):
print(input_layer)
conv1 = Conv1D(100, (3), activation='relu')(input_layer)
dropout_1 = Dropout(dropout_rate)(conv1)
conv2 = Conv1D(100, (4), activation='relu')(input_layer)
dropout_2 = Dropout(dropout_rate)(conv2)
conv3 = Conv1D(100, (5), activation='relu')(input_layer)
dropout_3 = Dropout(dropout_rate)(conv3)
conv4 = Conv1D(100, (6), activation='relu')(input_layer)
dropout_4 = Dropout(dropout_rate)(conv4)
maxpool1 = MaxPooling1D(pool_size=48)(dropout_1)
maxpool2 = MaxPooling1D(pool_size=47)(dropout_2)
maxpool3 = MaxPooling1D(pool_size=46)(dropout_3)
maxpool4 = MaxPooling1D(pool_size=45)(dropout_4)
return (maxpool1, maxpool2, maxpool3, maxpool4)
def convolution_and_max_pooling2(input_layer):
print(input_layer)
conv1 = Conv1D(100, (3), activation='relu')(input_layer)
dropout_1 = Dropout(dropout_rate)(conv1)
conv2 = Conv1D(100, (4), activation='relu')(input_layer)
dropout_2 = Dropout(dropout_rate)(conv2)
conv3 = Conv1D(100, (5), activation='relu')(input_layer)
dropout_3 = Dropout(dropout_rate)(conv3)
conv4 = Conv1D(100, (6), activation='relu')(input_layer)
dropout_4 = Dropout(dropout_rate)(conv4)
maxpool1 = MaxPooling1D(pool_size=33)(dropout_1)
maxpool2 = MaxPooling1D(pool_size=32)(dropout_2)
maxpool3 = MaxPooling1D(pool_size=31)(dropout_3)
maxpool4 = MaxPooling1D(pool_size=30)(dropout_4)
return (maxpool1, maxpool2, maxpool3, maxpool4)
max_pool_emb = convolution_and_max_pooling(embedding)
max_pool_pos = convolution_and_max_pooling2(pos_tagging)
cc1 = concatenate([
max_pool_emb[0], max_pool_emb[1], max_pool_emb[2], max_pool_emb[3],
max_pool_pos[0], max_pool_pos[1], max_pool_pos[2], max_pool_pos[3]
],
axis=2)
lstm = LSTM(300)(cc1)
flat_classifier = Flatten()(dense_3)
concatenation_layer = concatenate([lstm, flat_classifier])
output = Dense(1, activation="sigmoid")(concatenation_layer)
# concatenation_layer = concatenate(lstm)
model = Model(inputs=[deep_inputs, pos_tagging, other_features],
outputs=output)
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
return model
training_data, look_back)
validation_data_X, validation_data_Y = split_sequence_multivariate(
validation_data, look_back)
test_data_X, test_data_Y = split_sequence_multivariate(test_data, look_back)
training_data_X = training_data_X.reshape(training_data_X.shape[0],
training_data_X.shape[1], features)
validation_data_X = validation_data_X.reshape(validation_data_X.shape[0],
validation_data_X.shape[1],
features)
test_data_X = test_data_X.reshape(test_data_X.shape[0], test_data_X.shape[1],
features)
model = Sequential()
model.add(
Bidirectional(LSTM(100, activation='relu', input_shape=(look_back, 1))))
# model.add(Dense(100, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam',
loss='mse',
metrics=[metrics.mae, relative_squared_error])
train_history = model.fit(training_data_X,
training_data_Y,
epochs=500,
verbose=2,
validation_data=(validation_data_X,
validation_data_Y))
loss = train_history.history['loss']
val_loss = train_history.history['val_loss']
mae_loss = train_history.history['mean_absolute_error']
def create_CNN_LSTM_POS_model_attention(vocabulary_size, sequence_len,
embedding_matrix, EMBEDDING_SIZE,
pos_tag_list_len, len_features):
max_seq_length = sequence_len
deep_inputs = Input(shape=(max_seq_length, ))
# deep_inputs = Input(shape=max_seq_length)
print("deep input shape")
print(deep_inputs.shape)
embedding = Embedding(vocabulary_size,
EMBEDDING_SIZE,
input_length=sequence_len,
weights=[embedding_matrix],
trainable=False)(deep_inputs) # line A
pos_tagging = Input(shape=(pos_tag_list_len, 1))
other_features = Input(shape=(len_features, 1))
# dense_1 = Dense(16, activation="sigmoid")(other_features)
# dense_2 = Dense(8, activation="sigmoid")(dense_1)
# dense_3 = Dense(4, activation="sigmoid")(dense_2)
dense_1 = Dense(16, kernel_initializer='normal',
activation='relu')(other_features)
dense_2 = Dense(8, kernel_initializer='normal', activation='relu')(dense_1)
dense_3 = Dense(4, kernel_initializer='normal', activation='relu')(dense_2)
dropout_rate = 0.5
def convolution_and_max_pooling(input_layer):
print(input_layer)
conv1 = Conv1D(100, (3), activation='relu')(input_layer)
dropout_1 = Dropout(dropout_rate)(conv1)
conv2 = Conv1D(100, (4), activation='relu')(input_layer)
dropout_2 = Dropout(dropout_rate)(conv2)
conv3 = Conv1D(100, (5), activation='relu')(input_layer)
dropout_3 = Dropout(dropout_rate)(conv3)
conv4 = Conv1D(100, (6), activation='relu')(input_layer)
dropout_4 = Dropout(dropout_rate)(conv4)
maxpool1 = MaxPooling1D(pool_size=sequence_len - 2)(dropout_1)
maxpool2 = MaxPooling1D(pool_size=sequence_len - 3)(dropout_2)
maxpool3 = MaxPooling1D(pool_size=sequence_len - 4)(dropout_3)
maxpool4 = MaxPooling1D(pool_size=sequence_len - 5)(dropout_4)
return (maxpool1, maxpool2, maxpool3, maxpool4)
def convolution_and_max_pooling2(input_layer):
print(input_layer)
conv1 = Conv1D(100, (3), activation='relu')(input_layer)
dropout_1 = Dropout(dropout_rate)(conv1)
conv2 = Conv1D(100, (4), activation='relu')(input_layer)
dropout_2 = Dropout(dropout_rate)(conv2)
conv3 = Conv1D(100, (5), activation='relu')(input_layer)
dropout_3 = Dropout(dropout_rate)(conv3)
conv4 = Conv1D(100, (6), activation='relu')(input_layer)
dropout_4 = Dropout(dropout_rate)(conv4)
maxpool1 = MaxPooling1D(pool_size=33)(dropout_1)
maxpool2 = MaxPooling1D(pool_size=32)(dropout_2)
maxpool3 = MaxPooling1D(pool_size=31)(dropout_3)
maxpool4 = MaxPooling1D(pool_size=30)(dropout_4)
return (maxpool1, maxpool2, maxpool3, maxpool4)
print("before conv")
print(pos_tagging.shape)
max_pool_emb = convolution_and_max_pooling(embedding)
max_pool_pos = convolution_and_max_pooling2(pos_tagging)
print("after conv")
print(max_pool_emb[0].shape)
cc1 = concatenate([
max_pool_emb[0], max_pool_emb[1], max_pool_emb[2], max_pool_emb[3],
max_pool_pos[0], max_pool_pos[1], max_pool_pos[2], max_pool_pos[3]
],
axis=2)
lstm = Bidirectional(LSTM(300, return_sequences=True))(cc1)
print("after lstm")
print(lstm.shape)
attention = AttLayer(300)(lstm)
flat_classifier = Flatten()(dense_3)
concatenation_layer = concatenate([attention, flat_classifier])
#output = Dense(1, activation="sigmoid")(concatenation_layer)
# output = Dense(1, kernel_initializer='normal', activation="linear")(concatenation_layer)
output = Dense(1, kernel_initializer='normal',
activation="linear")(concatenation_layer)
#concatenation_layer = concatenate(lstm)
model = Model(inputs=[deep_inputs, pos_tagging, other_features],
outputs=output)
# model = Model(inputs=[deep_inputs, pos_tagging], outputs=output)
#model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.compile(loss='mean_absolute_error',
optimizer=optimizer,
metrics=['mean_absolute_error'])
model.summary()
return model
del X['target']
y = [x > 0 for x in lc]
a = len(X)
s = int(a * .7)
trainX = X[:s]
trainy = y[:s]
test0X = X[s:]
test0y = y[s:]
trainX2 = np.asarray(trainX).reshape(
(trainX.shape[0], 1, trainX.shape[1]))
test0X = np.asarray(test0X).reshape(
(test0X.shape[0], 1, test0X.shape[1]))
keras.backend.clear_session()
monkey = Sequential()
monkey.add(
LSTM(LSTMwidth, input_shape=(trainX2.shape[1], trainX2.shape[2])))
monkey.add(Dense(2))
monkey.add(Activation('softmax'))
monkey.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
monkey.fit(trainX2,
np.asarray(trainy),
nb_epoch=30,
validation_data=(test0X, np.asarray(test0y)))
balances.append(1 - sum(test0y) / len(test0y))
preds = monkey.predict(test0X)
predBalances.append(
float(sum(x[0] > x[1] for x in preds)) / float(len(preds)))
rets = []
del monkey
print("BUILDING MODEL")
embedding_vecor_length = 32
input_layer = Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length)
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(Conv1D(filters=32, kernel_size=3, padding='same'))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.2))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(100))
branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(Conv1D(filters=32, kernel_size=4, padding='same'))
branch_4.add(Activation('relu'))
branch_4.add(MaxPooling1D(pool_size=2))
branch_4.add(Dropout(0.2))
branch_4.add(BatchNormalization())
branch_4.add(LSTM(100))
branch_5 = Sequential()
branch_5.add(input_layer)
branch_5.add(Conv1D(filters=32, kernel_size=5, padding='same'))
branch_5.add(Activation('relu'))
branch_5.add(MaxPooling1D(pool_size=2))
merged_vector = keras.layers.concatenate([encoded_a, encoded_b], axis=-1) # add a logistic regression on top prediction = Dense(1, activation='sigmod')(merged_vector) # We define a trainable model linking the tweet inputs into predictors model = Model(inputs=[tweet_a, tweet_b], outputs=predictions) model.complie(optimizer='rmsprop', loss = 'binary_crossentropy', metrics=['accuracy']) model.fit([data_a, data_b], labels, epoch=10) """ each layer is a node in the computation graph. With shared layer, this layer will correpond to multiple nodes in the computation graph. To get the output for each computation, you will need to supply an index. """ a = Input(shape(140, 256)) lstm = LSTM(32) encoded_a = lstm(a) assert ltsm.output==encoded_a a = Input(shape(140, 256)) b = Input(shape(140, 256)) lstm = LSTM(32) encoded_a = lstm(a) encoded_b = lstm(b) assert lstm.get_output_at(0)==encoded_a assert lstm.get_output_at(1)==encoded_b """ inception module (go deeper with convolution) """ input_img = Input(shape=(256,256,3)) tower_1 = Conv2D(64, (1,1), padding='same', activation='relu')(input_img) tower_1 = Conv2D(64, (3,3), padding='same', activation='relu')(tower_1)
weight_val = np.ones(len(labels_val))
#if re_weight:
# weight_val *= 0.472001959
# weight_val[labels_val == 0] = 1.309028344
#
# Define the model structure
# ----------------------------------------------------------------------------
embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
lstm_layer = Bidirectional(LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm))
#lstm_layer = LSTM(num_lstm, dropout=rate_drop_lstm)
sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences_1 = embedding_layer(sequence_1_input)
x1 = lstm_layer(embedded_sequences_1)
x1 = Dropout(rate_drop_dense)(x1)
x1 = BatchNormalization()(x1)
x1 = Dense(num_dense, activation=act)(x1)
x1 = Dropout(rate_drop_dense)(x1)
x1 = BatchNormalization()(x1)
preds = Dense(5, activation='softmax')(x1)
#preds = Dense(5, activation=act)(x1)
#
def test_functional_guide():
# MNIST
from keras.layers import Input, Dense, LSTM
from keras.models import Model
from keras.utils import np_utils
# this returns a tensor
inputs = Input(shape=(784,))
# a layer instance is callable on a tensor, and returns a tensor
x = Dense(64, activation='relu')(inputs)
x = Dense(64, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
# this creates a model that includes
# the Input layer and three Dense layers
model = Model(input=inputs, output=predictions)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# the data, shuffled and split between tran and test sets
X_train = np.random.random((100, 784))
Y_train = np.random.random((100, 10))
model.fit(X_train, Y_train, nb_epoch=2, batch_size=128)
assert model.inputs == [inputs]
assert model.outputs == [predictions]
assert model.input == inputs
assert model.output == predictions
assert model.input_shape == (None, 784)
assert model.output_shape == (None, 10)
# try calling the sequential model
inputs = Input(shape=(784,))
new_outputs = model(inputs)
new_model = Model(input=inputs, output=new_outputs)
new_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
##################################################
# multi-io
##################################################
tweet_a = Input(shape=(4, 25))
tweet_b = Input(shape=(4, 25))
# this layer can take as input a matrix
# and will return a vector of size 64
shared_lstm = LSTM(64)
# when we reuse the same layer instance
# multiple times, the weights of the layer
# are also being reused
# (it is effectively *the same* layer)
encoded_a = shared_lstm(tweet_a)
encoded_b = shared_lstm(tweet_b)
# we can then concatenate the two vectors:
merged_vector = merge([encoded_a, encoded_b],
mode='concat', concat_axis=-1)
# and add a logistic regression on top
predictions = Dense(1, activation='sigmoid')(merged_vector)
# we define a trainable model linking the
# tweet inputs to the predictions
model = Model(input=[tweet_a, tweet_b], output=predictions)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
data_a = np.random.random((1000, 4, 25))
data_b = np.random.random((1000, 4, 25))
labels = np.random.random((1000,))
model.fit([data_a, data_b], labels, nb_epoch=1)
model.summary()
assert model.inputs == [tweet_a, tweet_b]
assert model.outputs == [predictions]
assert model.input == [tweet_a, tweet_b]
assert model.output == predictions
assert model.output == predictions
assert model.input_shape == [(None, 4, 25), (None, 4, 25)]
assert model.output_shape == (None, 1)
assert shared_lstm.get_output_at(0) == encoded_a
assert shared_lstm.get_output_at(1) == encoded_b
assert shared_lstm.input_shape == (None, 4, 25)
def get_model_lstm_conv2d(input_profile_names, target_profile_names, scalar_input_names,
actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):
profile_inshape = (profile_lookback, profile_length)
actuator_inshape = (actuator_lookback + lookahead,)
num_profiles = len(input_profile_names)
num_targets = len(target_profile_names)
num_actuators = len(actuator_names)
max_channels = 32
profile_inputs = []
profiles = []
for i in range(num_profiles):
profile_inputs.append(
Input(profile_inshape, name='input_' + input_profile_names[i]))
profiles.append(Reshape((profile_lookback, profile_length, 1))
(profile_inputs[i]))
profiles = Concatenate(axis=-1)(profiles)
# shape = (lookback, length, channels=num_profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/8), kernel_size=(1, 5),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels/4), kernel_size=(1, 10),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, 15),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
# shape = (lookback, length, channels)
if profile_lookback > 1:
profiles = Reshape((profile_lookback, 1, profile_length,
int(num_profiles*max_channels)))(profiles)
# shape = (lookback, 1, length, channels)
profiles = ConvLSTM2D(filters=int(num_profiles*max_channels), kernel_size=(10, 1),
strides=(1, 1), padding='same', activation=std_activation,
recurrent_activation='hard_sigmoid')(profiles)
#shape = (1, length, channels)
profiles = Reshape((profile_length, int(
num_profiles*max_channels)))(profiles)
# shape = (length, channels)
else:
profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, 10),
strides=(1, 1), padding='same', activation=std_activation)(profiles)
profiles = Reshape((profile_length, int(
num_profiles*max_channels)))(profiles)
# shape = (length, channels)
actuator_inputs = []
actuators = []
for i in range(num_actuators):
actuator_inputs.append(
Input(actuator_inshape, name='input_' + actuator_names[i]))
actuators.append(
Reshape((actuator_lookback+lookahead, 1))(actuator_inputs[i]))
actuators = Concatenate(axis=-1)(actuators)
# shaoe = (time, num_actuators)
actuators = Dense(units=int(num_profiles*max_channels/8),
activation=std_activation)(actuators)
actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
padding='causal', activation=std_activation)(actuators)
actuators = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
recurrent_activation='hard_sigmoid')(actuators)
actuators = Reshape((int(num_profiles*max_channels), 1))(actuators)
# shape = (channels, 1)
actuators = Dense(units=profile_length,
activation=std_activation)(actuators)
actuators = Dense(units=profile_length, activation=None)(actuators)
# shape = (channels, profile_length)
actuators = Permute(dims=(2, 1))(actuators)
# shape = (profile_length, channels)
merged = Add()([profiles, actuators])
merged = Reshape((1, profile_length, int(
num_profiles*max_channels)))(merged)
# shape = (1, length, channels)
prof_act = []
for i in range(num_targets):
prof_act.append(Conv2D(filters=max_channels, kernel_size=(1, 15), strides=(1, 1),
padding='same', activation=std_activation)(merged))
# shape = (1,length,max_channels)
prof_act[i] = Conv2D(filters=int(max_channels/4), kernel_size=(1, 15),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=int(max_channels/8), kernel_size=(1, 10),
strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
prof_act[i] = Conv2D(filters=1, kernel_size=(1, 5), strides=(1, 1),
padding='same', activation=None)(prof_act[i])
# shape = (1,length,1)
prof_act[i] = Reshape((profile_length,), name='target_' +
target_profile_names[i])(prof_act[i])
model = Model(inputs=profile_inputs + actuator_inputs, outputs=prof_act)
return model
# check if we are beyond the sequence if end_ix > len(sequence)-1: break # gather input and output parts of the pattern seq_x, seq_y = sequence[i:end_ix], sequence[end_ix] X.append(seq_x) y.append(seq_y) return array(X), array(y) # define input sequence raw_seq = [10, 20, 30, 40, 50, 60, 70, 80, 90] # choose a number of time steps n_steps = 3 # split into samples X, y = split_sequence(raw_seq, n_steps) # reshape from [samples, timesteps] into [samples, timesteps, features] n_features = 1 X = X.reshape((X.shape[0], X.shape[1], n_features)) # define model model = Sequential() model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, n_features))) model.add(LSTM(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse') # fit model model.fit(X, y, epochs=200, verbose=0) # demonstrate prediction x_input = array([70, 80, 90]) x_input = x_input.reshape((1, n_steps, n_features)) yhat = model.predict(x_input, verbose=0) print(yhat)
x2_1 = x2
# split
x1_train, x1_test, y1_train, y1_test = train_test_split(
x1, y1, train_size =0.8, random_state =30)
x2_train, x2_test, y2_train, y2_test = train_test_split(
x2, y2, train_size =0.8, random_state =30)
np.save('D:/Study/test/x1_data5.npy', arr = x1)
np.save('D:/Study/test/x2_data5.npy', arr = x2)
np.save('D:/Study/test/y_data5.npy', arr = y1)
#2. model
# 1
input1 = Input(shape = (5,1 ))
dense1 = LSTM(50, activation = 'elu')(input1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(100, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(150, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(200, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(250, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
# 2
input2 = Input(shape = (5, 5))
dense2 = LSTM(100, activation = 'elu')(input2)
dense2 = Dropout(0.2)(dense2)
dense2 = Dense(150, activation = 'elu')(dense2)
int(x_train.shape[1] / features), features])
x_test = x_test.reshape(
[x_test.shape[0],
int(x_test.shape[1] / features), features])
#importing keras model and layers to construct LSTM model
from keras.models import Sequential
from keras.layers import Dense, Flatten, LSTM, Dropout
#initializing regression model
regressor = Sequential()
#adding layer(s) to model
regressor.add(
LSTM(units=50,
return_sequences=True,
input_shape=(x_train.shape[1], x_train.shape[2])))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=50, return_sequences=True))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=33, return_sequences=True))
regressor.add(Flatten())
regressor.add(Dense(units=1))
#compiling the model with mean_absolute_percentage_error and adam optimizer
regressor.compile(optimizer='adam', loss='mean_absolute_percentage_error')
#fitting model with training sets and validation set
history = regressor.fit(x_train,
y_train,
for i in range(0, n_chars - seq_length, 1):
seq_in = raw_text[i:i + seq_length]
seq_out = raw_text[i + seq_length]
dataX.append([char_to_int[char] for char in seq_in])
dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print "Total Patterns: ", n_patterns
# reshape X to be [samples, time steps, features]
X = numpy.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# define the LSTM model
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
# load the network weights
filename = "weights-improvement-19-1.9435.hdf5"
model.load_weights(filename)
model.compile(loss='categorical_crossentropy', optimizer='adam')
# pick a random seed
start = numpy.random.randint(0, len(dataX) - 1)
pattern = dataX[start]
print "Seed:"
print "\"", ''.join([int_to_char[value] for value in pattern]), "\""
# generate characters
for i in range(1000):
x = numpy.reshape(pattern, (1, len(pattern), 1))
x = x / float(n_vocab)