示例#1
0
class LSTMSequenceEmbedder(SequenceEmbedder):
    """Forward LSTM Sequence Embedder

    Also provide attention states.
    """
    def __init__(self, token_embeds, seq_length, align='left', name='LSTMSequenceEmbedder', hidden_size=50):
        self.hidden_size = hidden_size
        super(LSTMSequenceEmbedder, self).__init__(token_embeds, align=align, seq_length=seq_length, name=name)

    def embed_sequences(self, embed_sequence_batch):
        self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
        # Pass input through the LSTMs
        # Shape: (batch_size, seq_length, hidden_size)
        hidden_state_values = self._forward_lstm(embed_sequence_batch.values, embed_sequence_batch.mask)
        self._hidden_states = SequenceBatch(hidden_state_values, embed_sequence_batch.mask)

        # Embedding dimension: (batch_size, hidden_size)
        shape = tf.shape(embed_sequence_batch.values)
        forward_final = tf.slice(hidden_state_values, [0, shape[1] - 1, 0], [-1, 1, self.hidden_size])
        return tf.squeeze(forward_final, [1])

    @property
    def weights(self):
        return self._forward_lstm.get_weights()

    @weights.setter
    def weights(self, w):
        self._forward_lstm.set_weights(w)

    @property
    def hidden_states(self):
        return self._hidden_states
示例#2
0
 def embed_sequences(self, embed_sequence_batch):
     """Return sentence embeddings as a tensor with with shape
     [batch_size, hidden_size * 2]
     """
     forward_values = embed_sequence_batch.values
     forward_mask = embed_sequence_batch.mask
     backward_values = tf.reverse(forward_values, [False, True, False])
     backward_mask = tf.reverse(forward_mask, [False, True])
     # Initialize LSTMs
     self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
     self._backward_lstm = LSTM(self.hidden_size, return_sequences=True)
     # Pass input through the LSTMs
     # Shape: (batch_size, seq_length, hidden_size)
     forward_seq = self._forward_lstm(forward_values, forward_mask)
     forward_seq.set_shape((None, self.seq_length, self.hidden_size))
     backward_seq = self._backward_lstm(backward_values, backward_mask)
     backward_seq.set_shape((None, self.seq_length, self.hidden_size))
     # Stitch the outputs together --> hidden states (for computing attention)
     # Final dimension: (batch_size, seq_length, hidden_size * 2)
     lstm_states = tf.concat(2, [forward_seq, tf.reverse(backward_seq, [False, True, False])])
     self._hidden_states = SequenceBatch(lstm_states, forward_mask)
     # Stitch the final outputs together --> sequence embedding
     # Final dimension: (batch_size, hidden_size * 2)
     seq_length = tf.shape(forward_values)[1]
     forward_final = tf.slice(forward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
     backward_final = tf.slice(backward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
     return tf.squeeze(tf.concat(2, [forward_final, backward_final]), [1])
def train_keras_nn():
    r = 1e-3
    graph = Graph()
    # graph.add_input(name='input_features', input_shape=(input_features.shape[1],))
    graph.add_input(name='train_product_nnid', input_shape=(1,), dtype=int)
    graph.add_input(name='query_indexes', input_shape=(15,), dtype=int)
    graph.add_node(Embedding(input_dim=len(prod) + 1,
                             output_dim=100,
                             weights=[np.concatenate((np.zeros((1, 100)), product_vectors), axis=0)],
                             trainable=True,
                             input_length=1,
                             W_regularizer=l2(r)),
                   name='embedding',
                   input='train_product_nnid', )
    graph.add_node(Flatten(), name='flatten', input='embedding')

    graph.add_node(Embedding(input_dim=len(vocab_vectors) + 1,
                             output_dim=50,
                             weights=[np.concatenate((np.zeros((1, 50)), vocab_vectors), axis=0)],
                             trainable=True,
                             input_length=1,
                             W_regularizer=l2(r), mask_zero=True),
                   name='q_embedding',
                   input='query_indexes', )
    lstm = LSTM(output_dim=50)
    graph.add_node(lstm, name='LSTM', input='q_embedding')
    graph.add_node(Dense(100, activation='sigmoid', W_regularizer=l2(r)), name='hidden0',
                   inputs=['LSTM', 'flatten'],
                   merge_mode="concat", concat_axis=1)
    graph.add_node(Dense(1, activation='sigmoid', W_regularizer=l2(r)), name='output', input='hidden0',
                   create_output=True)

    graph.compile(optimizer='adam', loss={'output': 'mse'})

    select_best = SelectBestValidation(graph)

    # To get weights of query vectors
    get_lstm_output = theano.function([graph.inputs['query_indexes'].input], lstm.get_output(train=False))

    graph.fit(
        {'train_product_nnid': np.concatenate([train_product_nnid, test_product_nnid]),
         'query_indexes': np.concatenate([query_train, query_test]),
         'output': (output - 1) / 2},
        batch_size=5000, nb_epoch=30, verbose=1, shuffle=True, callbacks=[select_best], validation_split=0.1)

    return graph, get_lstm_output
示例#4
0
    def embed_sequences(self, embed_sequence_batch):
        self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
        # Pass input through the LSTMs
        # Shape: (batch_size, seq_length, hidden_size)
        hidden_state_values = self._forward_lstm(embed_sequence_batch.values, embed_sequence_batch.mask)
        self._hidden_states = SequenceBatch(hidden_state_values, embed_sequence_batch.mask)

        # Embedding dimension: (batch_size, hidden_size)
        shape = tf.shape(embed_sequence_batch.values)
        forward_final = tf.slice(hidden_state_values, [0, shape[1] - 1, 0], [-1, 1, self.hidden_size])
        return tf.squeeze(forward_final, [1])
示例#5
0
class BidiLSTMSequenceEmbedder(SequenceEmbedder):
    """Bidirectional LSTM Sequence Embedder

    Also provide attention states.
    """
    def __init__(self, token_embeds, seq_length, align='left', name='BidiLSTMSequenceEmbedder', hidden_size=50):
        self.seq_length = seq_length
        self.hidden_size = hidden_size
        super(BidiLSTMSequenceEmbedder, self).__init__(token_embeds, align=align, seq_length=seq_length, name=name)

    def embed_sequences(self, embed_sequence_batch):
        """Return sentence embeddings as a tensor with with shape
        [batch_size, hidden_size * 2]
        """
        forward_values = embed_sequence_batch.values
        forward_mask = embed_sequence_batch.mask
        backward_values = tf.reverse(forward_values, [False, True, False])
        backward_mask = tf.reverse(forward_mask, [False, True])
        # Initialize LSTMs
        self._forward_lstm = LSTM(self.hidden_size, return_sequences=True)
        self._backward_lstm = LSTM(self.hidden_size, return_sequences=True)
        # Pass input through the LSTMs
        # Shape: (batch_size, seq_length, hidden_size)
        forward_seq = self._forward_lstm(forward_values, forward_mask)
        forward_seq.set_shape((None, self.seq_length, self.hidden_size))
        backward_seq = self._backward_lstm(backward_values, backward_mask)
        backward_seq.set_shape((None, self.seq_length, self.hidden_size))
        # Stitch the outputs together --> hidden states (for computing attention)
        # Final dimension: (batch_size, seq_length, hidden_size * 2)
        lstm_states = tf.concat(2, [forward_seq, tf.reverse(backward_seq, [False, True, False])])
        self._hidden_states = SequenceBatch(lstm_states, forward_mask)
        # Stitch the final outputs together --> sequence embedding
        # Final dimension: (batch_size, hidden_size * 2)
        seq_length = tf.shape(forward_values)[1]
        forward_final = tf.slice(forward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
        backward_final = tf.slice(backward_seq, [0, seq_length - 1, 0], [-1, 1, self.hidden_size])
        return tf.squeeze(tf.concat(2, [forward_final, backward_final]), [1])

    @property
    def weights(self):
        return (self._forward_lstm.get_weights(), self._backward_lstm.get_weights())

    @weights.setter
    def weights(self, w):
        forward_weights, backward_weights = w
        self._forward_lstm.set_weights(forward_weights)
        self._backward_lstm.set_weights(backward_weights)

    @property
    def hidden_states(self):
        """Return a SequenceBatch whose value has shape
        [batch_size, max_seq_length, hidden_size * 2]
        """
        return self._hidden_states
示例#6
0
    seq_in = raw_text[i:i + seq_length]
    seq_out = raw_text[i + seq_length]
    dataX.append([char_to_int[char] for char in seq_in])
    dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print "Total Patterns: ", n_patterns
# reshape X to be [samples, time steps, features]
X = numpy.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# define the LSTM model
model = Sequential()
model.add(
    LSTM(512, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
#model.add(LSTM(256, return_sequences=True))
#model.add(Dropout(0.2))
#model.add(LSTM(128, return_sequences=True))
#model.add(Dropout(0.2))
model.add(LSTM(256))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
# load the network weights
#filename = "weights/weights-improvement-most-recent.hdf5"
#filename = "weights/multilayer-weights.hdf5"
filename = "weights/sherlock-4-2l-512n-1000e-128bs-02do-507.hdf5"
model.load_weights(filename)
model.compile(loss='categorical_crossentropy',
              optimizer=Adam(lr=0.1,
示例#7
0
 def step(self, x, states):
     wx = x[:, :self.da_dim]
     da = x[:, self.da_dim:]
     h, states = LSTM.step(self, wx, states)
     states[1] = states[1] + self.activation(da) # c
     return h, states[:2]
示例#8
0
embedding_matrix = np.zeros((len(word_index) + 1, embed_size))
absent_words = 0
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector
    else:
        absent_words += 1

# -----------------embedding layer---------------------------------------------------
embedding_layer = Embedding(len(word_index) + 1, embed_size, weights=[embedding_matrix], input_length=max_senten_len,
                            trainable=False)
word_input = Input(shape=(max_senten_len,), dtype='float32')
word_sequences = embedding_layer(word_input)
word_lstm = Bidirectional(LSTM(HAN_LSTM_UNITS, return_sequences=True, kernel_regularizer=l2_reg))(word_sequences)
word_dense = TimeDistributed(Dense(4, kernel_regularizer=l2_reg))(word_lstm)
word_att = AttentionWithContext()(word_dense)
wordEncoder = Model(word_input, word_att)

sent_input = Input(shape=(max_senten_num, max_senten_len), dtype='float32')
sent_encoder = TimeDistributed(wordEncoder)(sent_input)
sent_lstm = Bidirectional(LSTM(HAN_LSTM_UNITS, return_sequences=True, kernel_regularizer=l2_reg))(sent_encoder)
sent_dense = TimeDistributed(Dense(4, kernel_regularizer=l2_reg))(sent_lstm)
sent_att = Dropout(DROP_OUT_HAN)(AttentionWithContext()(sent_dense))
preds = Dense(1, activation='sigmoid')(sent_att)
model = Model(sent_input, preds)
lr = 0.01
optimizer = optimizers.Adam(lr)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
if args.command == 'train':
示例#9
0
    y_train.append(training_set_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)

X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))

X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))

from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dropout
from keras.layers import Dense

model = Sequential()

model.add(
    LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], 1)))
model.add(Dropout(0.2))

model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))

model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))

model.add(LSTM(units=50))
model.add(Dropout(0.2))

model.add(Dense(units=1))

model.compile(optimizer='adam', loss='mean_squared_error')
示例#10
0
#2. 모델 구성
# model = Sequential()
# # model.add(LSTM(10, activation = 'relu', input_shape = (3,2))) # 실질적으로 행은 큰 영향을 안미치기 때문에 무시 몇개의 칼럼을 가지고 몇개씩 작업을 할 것인가
# model.add(LSTM(130, input_length = 3, input_dim = 1))
# model.add(Dense(149, activation= 'sigmoid'))
# model.add(Dense(35))
# model.add(Dense(60))
# model.add(Dense(90))
# model.add(Dense(690))
# model.add(Dense(610))
# model.add(Dense(470))
# model.add(Dense(250))
# model.add(Dense(1))

input1 = Input(shape=(3, 1))
dense1 = LSTM(20)(input1)
dense1 = Dense(420)(dense1)
output1 = Dense(1)(dense1)

model = Model(input=input1, output=output1)

model.summary()

# 실행
from keras.callbacks import EarlyStopping

earlystopping = EarlyStopping(monitor='loss', patience=10, mode='min')
model.compile(optimizer='adam', loss='mse')
model.fit(x, y, epochs=1000, batch_size=10, callbacks=[earlystopping])

x_predict = array([50, 60, 70])
num_batches = 300
batch_size=20
batch_length=20

batch_X,batch_Y = batchify(categorized,categorizedY,num_batches,batch_size,batch_length)

#debug examples...
#char_X = [char_indices[c] for c in X]
#char_Y = [char_indices[c] for c in Y]
#batch_X,batch_Y = batchify(np.asarray([char_indices[c] for c in X]).reshape(len(X),1),np.asarray([char_indices[c] for c in Y]).reshape(len(Y),1),10,10,10)


# create and fit the LSTM network
print('Building the Model layers')
model = Sequential()
model.add(LSTM(80, return_sequences=True,input_shape=(batch_size,len(chars)),stateful=True,batch_size=batch_size))
model.add(Dropout(0.2))
model.add(TimeDistributed(Dense(len(chars))))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
char_indices = dict((c, i) for i, c in enumerate(chars))

print('Starting to learn:')
for j in range(5):
    model.reset_states()
    for i in range(num_batches):
        print('{} out of {}, "epoch": {}'.format(i,num_batches,j))
        model.train_on_batch(batch_X[i], batch_Y[i])
        #test on batch for evaluation
def get_model_conv2d(input_profile_names, target_profile_names, scalar_input_names,
                     actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):

    max_profile_lookback = 0
    for sig in input_profile_names:
        if lookbacks[sig] > max_profile_lookback:
            max_profile_lookback = lookbacks[sig]
    max_actuator_lookback = 0
    for sig in actuator_names:
        if lookbacks[sig] > max_actuator_lookback:
            max_actuator_lookback = lookbacks[sig]
    max_scalar_lookback = 0
    for sig in scalar_input_names:
        if lookbacks[sig] > max_scalar_lookback:
            max_scalar_lookback = lookbacks[sig]

    num_profiles = len(input_profile_names)
    num_targets = len(target_profile_names)
    num_actuators = len(actuator_names)
    num_scalars = len(scalar_input_names)
    if 'max_channels' in kwargs:
        max_channels = kwargs['max_channels']
    else:
        max_channels = 32

    profile_inputs = []
    profiles = []
    for i in range(num_profiles):
        profile_inputs.append(
            Input((lookbacks[input_profile_names[i]], profile_length), name='input_' + input_profile_names[i]))
        profiles.append(Reshape((max_profile_lookback, profile_length, 1))
                        (ZeroPadding1D(padding=(max_profile_lookback - lookbacks[input_profile_names[i]], 0))(profile_inputs[i])))

    profiles = Concatenate(axis=-1)(profiles)
    # shape = (lookback, length, channels=num_profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels/8), kernel_size=(1, int(profile_length/12)),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels/4), kernel_size=(1, int(profile_length/8)),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels/2), kernel_size=(1, int(profile_length/6)),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, int(profile_length/4)),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    # shape = (lookback, length, channels)
    if max_profile_lookback > 1:
        profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(profile_lookback, 1),
                          strides=(1, 1), padding='valid', activation=std_activation)(profiles)
    profiles = Reshape((profile_length, int(
        num_profiles*max_channels)))(profiles)
    # shape = (length, channels)

    if num_scalars > 0:
        scalar_inputs = []
        scalars = []
        for i in range(num_scalars):
            scalar_inputs.append(
                Input((lookbacks[scalar_input_names[i]],), name='input_' + scalar_input_names[i]))
            scalars.append(Reshape((lookbacks[scalar_input_names[i]],1))(scalar_inputs[i]))
            scalars[i] = ZeroPadding1D(padding=(max_scalar_lookback - lookbacks[scalar_input_names[i]], 0))(scalars[i])
        scalars = Concatenate(axis=-1)(scalars)
        # shaoe = (time, num_actuators)
        scalars = Dense(units=int(num_profiles*max_channels/8),
                        activation=std_activation)(scalars)
        # actuators = Conv1D(filters=int(num_profiles*max_channels/8), kernel_size=3, strides=1,
        #                    padding='causal', activation=std_activation)(actuators)
        scalars = Dense(units=int(num_profiles*max_channels/4),
                        activation=std_activation)(scalars)
        # actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
        #                    padding='causal', activation=std_activation)(actuators)
        scalars = Dense(units=int(num_profiles*max_channels/2),
                        activation=std_activation)(scalars)
        scalars = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
                       recurrent_activation='hard_sigmoid')(scalars)
        scalars = Reshape((int(num_profiles*max_channels), 1))(scalars)
        # shape = (channels, 1)
        scalars = Dense(units=int(profile_length/4),
                        activation=std_activation)(scalars)
        scalars = Dense(units=int(profile_length/2),
                        activation=std_activation)(scalars)
        scalars = Dense(units=profile_length, activation=None)(scalars)
        # shape = (channels, profile_length)
        scalars = Permute(dims=(2, 1))(scalars)
        # shape = (profile_length, channels)

    actuator_future_inputs = []
    actuator_past_inputs = []
    actuators = []
    for i in range(num_actuators):
        actuator_future_inputs.append(
            Input((lookahead, ), name='input_future_' + actuator_names[i]))
        actuator_past_inputs.append(
            Input((lookbacks[actuator_names[i]], ), name='input_past_' + actuator_names[i]))
        actuators.append(Concatenate(
            axis=1)([actuator_past_inputs[i], actuator_future_inputs[i]]))
        actuators[i] = Reshape(
            (lookbacks[actuator_names[i]]+lookahead, 1))(actuators[i])
        actuators[i] = ZeroPadding1D(padding=(max_actuator_lookback - lookbacks[actuator_names[i]], 0))(actuators[i])
    actuators = Concatenate(axis=-1)(actuators)
    # shaoe = (time, num_actuators)
    actuators = Dense(units=int(num_profiles*max_channels/8),
                      activation=std_activation)(actuators)
    # actuators = Conv1D(filters=int(num_profiles*max_channels/8), kernel_size=3, strides=1,
    #                    padding='causal', activation=std_activation)(actuators)
    actuators = Dense(units=int(num_profiles*max_channels/4),
                      activation=std_activation)(actuators)
    # actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
    #                    padding='causal', activation=std_activation)(actuators)
    actuators = Dense(units=int(num_profiles*max_channels/2),
                      activation=std_activation)(actuators)
    actuators = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
                     recurrent_activation='hard_sigmoid')(actuators)
    actuators = Reshape((int(num_profiles*max_channels), 1))(actuators)
    # shape = (channels, 1)
    actuators = Dense(units=int(profile_length/4),
                      activation=std_activation)(actuators)
    actuators = Dense(units=int(profile_length/2),
                      activation=std_activation)(actuators)
    actuators = Dense(units=profile_length, activation=None)(actuators)
    # shape = (channels, profile_length)
    actuators = Permute(dims=(2, 1))(actuators)
    # shape = (profile_length, channels)

    if num_scalars > 0:
        merged = Add()([profiles, actuators, scalars])
    else:
        merged = Add()([profiles, actuators])
    merged = Reshape((1, profile_length, int(
        num_profiles*max_channels)))(merged)
    # shape = (1, length, channels)

    prof_act = []
    for i in range(num_targets):
        prof_act.append(Conv2D(filters=max_channels, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
                               padding='same', activation=std_activation)(merged))
        # shape = (1,length,max_channels)
        prof_act[i] = Conv2D(filters=int(max_channels/2), kernel_size=(1, int(profile_length/8)),
                             strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
        prof_act[i] = Conv2D(filters=int(max_channels/4), kernel_size=(1, int(profile_length/6)),
                             strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
        prof_act[i] = Conv2D(filters=int(max_channels/8), kernel_size=(1, int(profile_length/4)),
                             strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
        prof_act[i] = Conv2D(filters=1, kernel_size=(1, int(profile_length/4)), strides=(1, 1),
                             padding='same', activation=None)(prof_act[i])
        # shape = (1,length,1)
        if kwargs.get('predict_mean'):
            prof_act[i] = GlobalAveragePooling2D()(prof_act[i])
        else:
            prof_act[i] = Reshape((profile_length,), name='target_' +
                                  target_profile_names[i])(prof_act[i])

    model_inputs = profile_inputs + actuator_past_inputs + actuator_future_inputs
    if num_scalars > 0:
        model_inputs += scalar_inputs
    model_outputs = prof_act
    model = Model(inputs=model_inputs, outputs=model_outputs)
    return model
示例#13
0
test_size = length - train_size
batch_size_array = [8, 16, 32, 64, 128]
trainX, trainY = data[0:train_size], CPU_nomal[0:train_size]
testX = data[train_size:length]
testY = dataset.T[1][train_size:length]
# reshape input to be [samples, time steps, features]

trainX = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
testX = np.reshape(testX, (testX.shape[0], testX.shape[1], 1))

# create and fit the LSTM network
for batch_size in batch_size_array:
    print "batch_size= ", batch_size
    model = Sequential()
    model.add(
        LSTM(512, return_sequences=True, activation='relu',
             input_shape=(2, 1)))
    model.add(LSTM(4))
    model.add(Dense(1))
    model.compile(loss='mean_squared_error',
                  optimizer='adam',
                  metrics=['mean_squared_error'])
    history = model.fit(trainX,
                        trainY,
                        epochs=2000,
                        batch_size=batch_size,
                        verbose=2,
                        validation_split=0.1,
                        callbacks=[
                            EarlyStopping(monitor='loss',
                                          patience=20,
                                          verbose=1), tensorboard
x_train_titles = x_train.loc[:,x_train.columns.isin(['title_a', 'title_b'])]
x_train_emebeddings = x_train.loc[:,~x_train.columns.isin(['title_a', 'title_b'])]
x_test_titles = x_test.loc[:,x_train.columns.isin(['title_a', 'title_b'])]
x_test_emebeddings = x_test.loc[:,~x_train.columns.isin(['title_a', 'title_b'])]

'''


GLOVE_EMBEDDING_DIMS = 100

a_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
b_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')

shared_model = Sequential()
shared_model.add(Embedding(num_words, GLOVE_EMBEDDING_DIMS, input_length=MAX_SEQUENCE_LENGTH, weights= [embedding_matrix], trainable=False))
shared_model.add(LSTM(100))

# Overarching model.
a_features = shared_model(a_input)
b_features = shared_model(b_input)

siamese_model = concatenate([a_features, b_features])
siamese_model = Dropout(0.2)(siamese_model)
preds = Dense(1,  activation='sigmoid')(siamese_model)

model = Model(inputs=[a_input, b_input], outputs=preds)
model.compile(optimizer='adam', loss='mse', metrics=['mae', rmse])
"""
model = Sequential()
model.add(Embedding(num_words, GLOVE_EMBEDDING_DIMS, input_length=MAX_SEQUENCE_LENGTH, weights= [embedding_matrix], trainable=False))
model.add(Dropout(0.2))
    next_chars.append(text[i + maxlen])
print('nb sequences:', len(sentences))

print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1


# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)


def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)
示例#16
0
testY = dataY(test, look_back)



print('Conjunto de entrenamientoX t: \n' + str(trainX))
print('Con shape' + str(trainX.shape))
print('Conjunto de entrenamientoY t+1: \n' + str(trainY))
print('Con shape' + str(trainY.shape))
print('Conjunto pruebaX t: \n' + str(testX))
print('Conjunto pruebaY t+1: \n' + str(testY))



#Create and fit the LSTM network
model = Sequential()
model.add(LSTM(4, input_shape=(look_back,1)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')

es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15, min_delta=0.001)

history = model.fit(trainX, trainY, epochs=20, validation_split=0.3, batch_size=1, verbose=2, callbacks=[es])

#Represent loss
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
pyplot.show()

#Make prediction
trainPredict = model.predict(trainX)
示例#17
0
train_size = int(len(dataset) * 0.67)
test_size = len(dataset) - train_size
train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :]
# reshape into X=t and Y=t+1
look_back = 3
trainX, trainY = create_dataset(train, look_back)
testX, testY = create_dataset(test, look_back)
# reshape input to be [samples, time steps, features]
trainX = numpy.reshape(trainX, (trainX.shape[0], trainX.shape[1], 1))
testX = numpy.reshape(testX, (testX.shape[0], testX.shape[1], 1))
# create and fit the LSTM network
batch_size = 1
model = Sequential()
model.add(
    LSTM(4,
         batch_input_shape=(batch_size, look_back, 1),
         stateful=True,
         return_sequences=True))
model.add(LSTM(4, batch_input_shape=(batch_size, look_back, 1), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(100):
    model.fit(trainX,
              trainY,
              epochs=1,
              batch_size=batch_size,
              verbose=2,
              shuffle=False)
    model.reset_states()
# make predictions
trainPredict = model.predict(trainX, batch_size=batch_size)
model.reset_states()
示例#18
0
t = pickle.load(open('data/tokenizer.p', 'rb'))
Tx = 10
Ty = 10
m = X.shape[0]
repeator = RepeatVector(Tx)
concatenator = Concatenate(axis=-1)
densor1 = Dense(10, activation="tanh")
densor2 = Dense(1, activation="relu")
activator = Activation(
    softmax, name='attention_weights'
)  # We are using a custom softmax(axis = 1) loaded in this notebook
dotor = Dot(axes=1)

n_a = 32
n_s = 64
post_activation_LSTM_cell = LSTM(n_s, return_state=True)
output_layer = Dense(len(word_index) + 1, activation=softmax)

model = model(Tx, Ty, n_a, n_s, len(word_index) + 1, len(word_index) + 1)

opt = Adam(lr=0.008, beta_1=0.9, beta_2=0.999, decay=0.001)
model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])
model.summary()

s0 = np.zeros((m, n_s))
c0 = np.zeros((m, n_s))
outputs = list(Y.swapaxes(0, 1))

if os.path.isfile(
示例#19
0
print(y_train.shape)
print(y_train[0])

###################### build network ####################
######## word dim 词向量维度 ########
word_dim = 8

######## network structure ########
model = Sequential()

#### Embedding层 ####
model.add(Embedding(input_dim=1000, output_dim=word_dim, input_length=max_len))

#### 两层LSTM,第一层,设置return_sequences参数为True ####
model.add(LSTM(256, return_sequences=True))

#### dropout ####
model.add(Dropout(0.5))

#### 两层LSTM,第二层,设置return_sequences参数为False ####
model.add(LSTM(256, return_sequences=False))

#### dropout ####
model.add(Dropout(0.5))

#### 输出层 ####
model.add(Dense(num_class, activation='softmax'))

print(model.summary())
示例#20
0
# # Модель нейронной сети

# In[22]:

graph = tf.Graph()
with graph.as_default():
    input_X = tf.placeholder(tf.float32,
                             shape=[None, None, 13],
                             name="input_X")
    labels = tf.sparse_placeholder(tf.int32)
    seq_lens = tf.placeholder(tf.int32, shape=[None], name="seq_lens")

    model = Sequential()
    model.add(
        Bidirectional(LSTM(128, return_sequences=True, implementation=2),
                      input_shape=(None, 13)))
    model.add(Bidirectional(LSTM(128, return_sequences=True,
                                 implementation=2)))
    model.add(TimeDistributed(Dense(len(inv_mapping) + 2)))

    final_seq_lens = seq_lens

    logits = model(input_X)
    logits = tf.transpose(logits, [1, 0, 2])

    ctc_loss = tf.reduce_mean(tf.nn.ctc_loss(labels, logits, final_seq_lens))
    # ctc_greedy_decoder? merge_repeated=True
    decoded, log_prob = tf.nn.ctc_greedy_decoder(logits, final_seq_lens)
    ler = tf.reduce_mean(
        tf.edit_distance(tf.cast(decoded[0], tf.int32), labels))
def get_model_linear_systems(input_profile_names, target_profile_names, scalar_input_names,
                             actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):

    profile_inshape = (profile_lookback, profile_length)
    actuator_inshape = (actuator_lookback + lookahead,)
    num_profiles = len(input_profile_names)
    num_targets = len(target_profile_names)
    num_actuators = len(actuator_names)

    profile_inputs = []
    for i in range(num_profiles):
        profile_inputs.append(
            Input(profile_inshape, name='input_' + input_profile_names[i]))
    if num_profiles > 1:
        profiles = Concatenate(axis=-1)(profile_inputs)
    else:
        profiles = profile_inputs[0]
    profile_response = Dense(
        int(profile_length/2*num_profiles), activation=std_activation)(profiles)
    profile_response = Dense(
        int(profile_length/2*num_profiles), activation=std_activation)(profile_response)
    if profile_lookback > 1:
        profile_response = LSTM(int(profile_length/2*num_profiles), activation=std_activation,
                                recurrent_activation='hard_sigmoid',
                                return_sequences=True)(profile_response)
    else:
        profile_response = Dense(int(profile_length/2*num_profiles),
                                 activation=std_activation)(profile_response)
    profile_response = Dense(int(profile_length/2*num_profiles),
                             activation=std_activation)(profile_response)

    actuator_inputs = []
    actuators = []
    for i in range(num_actuators):
        actuator_inputs.append(
            Input(actuator_inshape, name='input_' + actuator_names[i]))
        actuators.append(
            Reshape((actuator_lookback+lookahead, 1))(actuator_inputs[i]))
    if num_actuators > 1:
        actuators = Concatenate(axis=-1)(actuators)
    else:
        actuators = actuators[0]

    actuator_response = Dense(
        profile_lookback, activation=std_activation)(actuators)
    actuator_response = Dense(
        profile_lookback, activation=std_activation)(actuator_response)
    actuator_response = LSTM(profile_lookback, activation=std_activation,
                             recurrent_activation='hard_sigmoid',
                             return_sequences=True)(actuator_response)
    total_response = Dot(axes=(2, 1))([actuator_response, profile_response])
    total_response = LSTM(int(profile_length/2*num_targets), activation=std_activation,
                          recurrent_activation='hard_sigmoid')(total_response)
    total_response = Dense(int(profile_length*.75*num_targets),
                           activation=std_activation)(total_response)
    total_response = Dense(profile_length*num_targets)(total_response)
    total_response = Reshape((num_targets*profile_length, 1))(total_response)

    targets = []
    for i in range(num_targets):
        targets.append(Cropping1D(cropping=(i*profile_length,
                                            (num_targets-i-1)*profile_length))(total_response))
        targets[i] = Reshape((profile_length,),
                             name='target_' + target_profile_names[i])(targets[i])
    model = Model(inputs=profile_inputs+actuator_inputs, outputs=targets)
    return model
示例#22
0
embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

embedding_layer = Embedding(len(word_index) + 1,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=False)

model = Sequential()
model.add(embedding_layer)
model.add(LSTM(LSTM_OUT, dropout_U=0.25, dropout_W=0.25))
model.add(Dense(NUM_CLASSES, activation='softmax'))
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])
print(model.summary())

checkpoint = ModelCheckpoint(weights,
                             monitor='val_acc',
                             verbose=1,
                             save_best_only=True,
                             save_weights_only=False,
                             mode='auto',
                             period=1)
early = EarlyStopping(monitor='val_acc',
                      min_delta=0,
hidden_units = 100
batch_size = 2
epochs = 100
learning_rate = 0.01

x_train = data_XX
y_train = data_YY

x_test = data_XX
y_test = data_YY

print("x_train.shape[1:] : ", x_train.shape[1:])

model = Sequential()
model.add(LSTM(hidden_units,
               return_sequences=True,
               input_shape=x_train.shape[1:]))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
rmsprop = RMSprop(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
              optimizer=rmsprop,
              metrics=['accuracy'])

model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          validation_data=(x_test, y_test))

scores = model.evaluate(x_test, y_test, verbose=1)
示例#24
0
print("number of words in vocab: ", len(vocab))
print("number of poems: ", len(poems))
print(len(seq_poems))
print(len(img_names))
print(len(features))
print("length of longest poem: ", max_poem_len)



inputs1 = Input(shape=(2048,))
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation='relu')(fe1)
inputs2 = Input(shape=(max_poem_len,))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
se3 = LSTM(256)(se2)
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
outputs = Dense(vocab_size, activation='softmax')(decoder2)
model = Model(inputs=[inputs1, inputs2], outputs=outputs)
        
model.layers[2].set_weights([embedding_matrix])
model.layers[2].trainable = False

model.compile(loss='categorical_crossentropy', optimizer='adam')

##############################

epochs = 2
number_pics_per_batch = 3
num_photos_per_batch = len(poems) #number_pics_per_batch
示例#25
0
    lst.append(lst.pop(0))
    lst.reverse()

print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')

print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)

print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print('Train...')
model.fit(x_train,
          y_train,
          batch_size=batch_size,
          epochs=15,
          validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
def create_CNN_LSTM_POS_model(vocabulary_size, sequence_len, embedding_matrix,
                              EMBEDDING_SIZE, pos_tag_list_len, len_features):
    max_seq_length = sequence_len
    deep_inputs = Input(shape=(max_seq_length, ))

    embedding = Embedding(vocabulary_size,
                          EMBEDDING_SIZE,
                          input_length=sequence_len,
                          weights=[embedding_matrix],
                          trainable=False)(deep_inputs)  # line A

    pos_tagging = Input(shape=(pos_tag_list_len, 1))

    other_features = Input(shape=(len_features, 1))

    dense_1 = Dense(4, activation="sigmoid")(other_features)
    dense_2 = Dense(8, activation="sigmoid")(dense_1)
    dense_3 = Dense(16, activation="sigmoid")(dense_2)

    dropout_rate = 0.5

    def convolution_and_max_pooling(input_layer):
        print(input_layer)
        conv1 = Conv1D(100, (3), activation='relu')(input_layer)
        dropout_1 = Dropout(dropout_rate)(conv1)
        conv2 = Conv1D(100, (4), activation='relu')(input_layer)
        dropout_2 = Dropout(dropout_rate)(conv2)
        conv3 = Conv1D(100, (5), activation='relu')(input_layer)
        dropout_3 = Dropout(dropout_rate)(conv3)
        conv4 = Conv1D(100, (6), activation='relu')(input_layer)
        dropout_4 = Dropout(dropout_rate)(conv4)
        maxpool1 = MaxPooling1D(pool_size=48)(dropout_1)
        maxpool2 = MaxPooling1D(pool_size=47)(dropout_2)
        maxpool3 = MaxPooling1D(pool_size=46)(dropout_3)
        maxpool4 = MaxPooling1D(pool_size=45)(dropout_4)
        return (maxpool1, maxpool2, maxpool3, maxpool4)

    def convolution_and_max_pooling2(input_layer):
        print(input_layer)
        conv1 = Conv1D(100, (3), activation='relu')(input_layer)
        dropout_1 = Dropout(dropout_rate)(conv1)
        conv2 = Conv1D(100, (4), activation='relu')(input_layer)
        dropout_2 = Dropout(dropout_rate)(conv2)
        conv3 = Conv1D(100, (5), activation='relu')(input_layer)
        dropout_3 = Dropout(dropout_rate)(conv3)
        conv4 = Conv1D(100, (6), activation='relu')(input_layer)
        dropout_4 = Dropout(dropout_rate)(conv4)
        maxpool1 = MaxPooling1D(pool_size=33)(dropout_1)
        maxpool2 = MaxPooling1D(pool_size=32)(dropout_2)
        maxpool3 = MaxPooling1D(pool_size=31)(dropout_3)
        maxpool4 = MaxPooling1D(pool_size=30)(dropout_4)
        return (maxpool1, maxpool2, maxpool3, maxpool4)

    max_pool_emb = convolution_and_max_pooling(embedding)
    max_pool_pos = convolution_and_max_pooling2(pos_tagging)

    cc1 = concatenate([
        max_pool_emb[0], max_pool_emb[1], max_pool_emb[2], max_pool_emb[3],
        max_pool_pos[0], max_pool_pos[1], max_pool_pos[2], max_pool_pos[3]
    ],
                      axis=2)

    lstm = LSTM(300)(cc1)
    flat_classifier = Flatten()(dense_3)
    concatenation_layer = concatenate([lstm, flat_classifier])
    output = Dense(1, activation="sigmoid")(concatenation_layer)
    # concatenation_layer = concatenate(lstm)
    model = Model(inputs=[deep_inputs, pos_tagging, other_features],
                  outputs=output)
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['accuracy'])
    model.summary()
    return model
示例#27
0
    training_data, look_back)
validation_data_X, validation_data_Y = split_sequence_multivariate(
    validation_data, look_back)
test_data_X, test_data_Y = split_sequence_multivariate(test_data, look_back)

training_data_X = training_data_X.reshape(training_data_X.shape[0],
                                          training_data_X.shape[1], features)
validation_data_X = validation_data_X.reshape(validation_data_X.shape[0],
                                              validation_data_X.shape[1],
                                              features)
test_data_X = test_data_X.reshape(test_data_X.shape[0], test_data_X.shape[1],
                                  features)

model = Sequential()
model.add(
    Bidirectional(LSTM(100, activation='relu', input_shape=(look_back, 1))))
# model.add(Dense(100, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam',
              loss='mse',
              metrics=[metrics.mae, relative_squared_error])

train_history = model.fit(training_data_X,
                          training_data_Y,
                          epochs=500,
                          verbose=2,
                          validation_data=(validation_data_X,
                                           validation_data_Y))
loss = train_history.history['loss']
val_loss = train_history.history['val_loss']
mae_loss = train_history.history['mean_absolute_error']
def create_CNN_LSTM_POS_model_attention(vocabulary_size, sequence_len,
                                        embedding_matrix, EMBEDDING_SIZE,
                                        pos_tag_list_len, len_features):
    max_seq_length = sequence_len
    deep_inputs = Input(shape=(max_seq_length, ))
    # deep_inputs = Input(shape=max_seq_length)
    print("deep input shape")
    print(deep_inputs.shape)
    embedding = Embedding(vocabulary_size,
                          EMBEDDING_SIZE,
                          input_length=sequence_len,
                          weights=[embedding_matrix],
                          trainable=False)(deep_inputs)  # line A

    pos_tagging = Input(shape=(pos_tag_list_len, 1))

    other_features = Input(shape=(len_features, 1))

    # dense_1 = Dense(16, activation="sigmoid")(other_features)
    # dense_2 = Dense(8, activation="sigmoid")(dense_1)
    # dense_3 = Dense(4, activation="sigmoid")(dense_2)
    dense_1 = Dense(16, kernel_initializer='normal',
                    activation='relu')(other_features)
    dense_2 = Dense(8, kernel_initializer='normal', activation='relu')(dense_1)
    dense_3 = Dense(4, kernel_initializer='normal', activation='relu')(dense_2)
    dropout_rate = 0.5

    def convolution_and_max_pooling(input_layer):
        print(input_layer)
        conv1 = Conv1D(100, (3), activation='relu')(input_layer)
        dropout_1 = Dropout(dropout_rate)(conv1)
        conv2 = Conv1D(100, (4), activation='relu')(input_layer)
        dropout_2 = Dropout(dropout_rate)(conv2)
        conv3 = Conv1D(100, (5), activation='relu')(input_layer)
        dropout_3 = Dropout(dropout_rate)(conv3)
        conv4 = Conv1D(100, (6), activation='relu')(input_layer)
        dropout_4 = Dropout(dropout_rate)(conv4)
        maxpool1 = MaxPooling1D(pool_size=sequence_len - 2)(dropout_1)
        maxpool2 = MaxPooling1D(pool_size=sequence_len - 3)(dropout_2)
        maxpool3 = MaxPooling1D(pool_size=sequence_len - 4)(dropout_3)
        maxpool4 = MaxPooling1D(pool_size=sequence_len - 5)(dropout_4)
        return (maxpool1, maxpool2, maxpool3, maxpool4)

    def convolution_and_max_pooling2(input_layer):
        print(input_layer)
        conv1 = Conv1D(100, (3), activation='relu')(input_layer)
        dropout_1 = Dropout(dropout_rate)(conv1)
        conv2 = Conv1D(100, (4), activation='relu')(input_layer)
        dropout_2 = Dropout(dropout_rate)(conv2)
        conv3 = Conv1D(100, (5), activation='relu')(input_layer)
        dropout_3 = Dropout(dropout_rate)(conv3)
        conv4 = Conv1D(100, (6), activation='relu')(input_layer)
        dropout_4 = Dropout(dropout_rate)(conv4)
        maxpool1 = MaxPooling1D(pool_size=33)(dropout_1)
        maxpool2 = MaxPooling1D(pool_size=32)(dropout_2)
        maxpool3 = MaxPooling1D(pool_size=31)(dropout_3)
        maxpool4 = MaxPooling1D(pool_size=30)(dropout_4)
        return (maxpool1, maxpool2, maxpool3, maxpool4)

    print("before conv")
    print(pos_tagging.shape)

    max_pool_emb = convolution_and_max_pooling(embedding)
    max_pool_pos = convolution_and_max_pooling2(pos_tagging)
    print("after conv")
    print(max_pool_emb[0].shape)

    cc1 = concatenate([
        max_pool_emb[0], max_pool_emb[1], max_pool_emb[2], max_pool_emb[3],
        max_pool_pos[0], max_pool_pos[1], max_pool_pos[2], max_pool_pos[3]
    ],
                      axis=2)

    lstm = Bidirectional(LSTM(300, return_sequences=True))(cc1)
    print("after lstm")
    print(lstm.shape)
    attention = AttLayer(300)(lstm)
    flat_classifier = Flatten()(dense_3)
    concatenation_layer = concatenate([attention, flat_classifier])

    #output = Dense(1, activation="sigmoid")(concatenation_layer)
    # output = Dense(1, kernel_initializer='normal', activation="linear")(concatenation_layer)
    output = Dense(1, kernel_initializer='normal',
                   activation="linear")(concatenation_layer)
    #concatenation_layer = concatenate(lstm)
    model = Model(inputs=[deep_inputs, pos_tagging, other_features],
                  outputs=output)
    # model = Model(inputs=[deep_inputs, pos_tagging], outputs=output)
    #model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])

    model.compile(loss='mean_absolute_error',
                  optimizer=optimizer,
                  metrics=['mean_absolute_error'])
    model.summary()
    return model
示例#29
0
 del X['target']
 y = [x > 0 for x in lc]
 a = len(X)
 s = int(a * .7)
 trainX = X[:s]
 trainy = y[:s]
 test0X = X[s:]
 test0y = y[s:]
 trainX2 = np.asarray(trainX).reshape(
     (trainX.shape[0], 1, trainX.shape[1]))
 test0X = np.asarray(test0X).reshape(
     (test0X.shape[0], 1, test0X.shape[1]))
 keras.backend.clear_session()
 monkey = Sequential()
 monkey.add(
     LSTM(LSTMwidth, input_shape=(trainX2.shape[1], trainX2.shape[2])))
 monkey.add(Dense(2))
 monkey.add(Activation('softmax'))
 monkey.compile(loss='sparse_categorical_crossentropy',
                optimizer='adam',
                metrics=['acc'])
 monkey.fit(trainX2,
            np.asarray(trainy),
            nb_epoch=30,
            validation_data=(test0X, np.asarray(test0y)))
 balances.append(1 - sum(test0y) / len(test0y))
 preds = monkey.predict(test0X)
 predBalances.append(
     float(sum(x[0] > x[1] for x in preds)) / float(len(preds)))
 rets = []
 del monkey
示例#30
0
print("BUILDING MODEL")
embedding_vecor_length = 32

input_layer = Embedding(top_words,
                        embedding_vecor_length,
                        input_length=max_review_length)

branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(Conv1D(filters=32, kernel_size=3, padding='same'))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.2))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(100))

branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(Conv1D(filters=32, kernel_size=4, padding='same'))
branch_4.add(Activation('relu'))
branch_4.add(MaxPooling1D(pool_size=2))
branch_4.add(Dropout(0.2))
branch_4.add(BatchNormalization())
branch_4.add(LSTM(100))

branch_5 = Sequential()
branch_5.add(input_layer)
branch_5.add(Conv1D(filters=32, kernel_size=5, padding='same'))
branch_5.add(Activation('relu'))
branch_5.add(MaxPooling1D(pool_size=2))
merged_vector = keras.layers.concatenate([encoded_a, encoded_b], axis=-1)

# add a logistic regression on top
prediction = Dense(1, activation='sigmod')(merged_vector)

# We define a trainable model linking the tweet inputs into predictors
model = Model(inputs=[tweet_a, tweet_b], outputs=predictions)
model.complie(optimizer='rmsprop', loss = 'binary_crossentropy', metrics=['accuracy'])
model.fit([data_a, data_b], labels, epoch=10)

""" each layer is a node in the computation graph. With shared layer, this
layer will correpond to multiple nodes in the computation graph. To get the
output for each computation, you will need to supply an index.
"""
a = Input(shape(140, 256))
lstm = LSTM(32)
encoded_a = lstm(a)
assert ltsm.output==encoded_a

a = Input(shape(140, 256))
b = Input(shape(140, 256))
lstm = LSTM(32)
encoded_a = lstm(a)
encoded_b = lstm(b)
assert lstm.get_output_at(0)==encoded_a
assert lstm.get_output_at(1)==encoded_b

""" inception module (go deeper with convolution) """
input_img = Input(shape=(256,256,3))
tower_1 = Conv2D(64, (1,1), padding='same', activation='relu')(input_img)
tower_1 = Conv2D(64, (3,3), padding='same', activation='relu')(tower_1)
示例#32
0
weight_val = np.ones(len(labels_val))
#if re_weight:
#    weight_val *= 0.472001959
#    weight_val[labels_val == 0] = 1.309028344


#
# Define the model structure
# ----------------------------------------------------------------------------
embedding_layer = Embedding(nb_words,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=True)
lstm_layer = Bidirectional(LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm))
#lstm_layer = LSTM(num_lstm, dropout=rate_drop_lstm)

sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences_1 = embedding_layer(sequence_1_input)
x1 = lstm_layer(embedded_sequences_1)
x1 =  Dropout(rate_drop_dense)(x1)
x1 =  BatchNormalization()(x1)

x1 = Dense(num_dense, activation=act)(x1)
x1 = Dropout(rate_drop_dense)(x1)
x1 = BatchNormalization()(x1)

preds = Dense(5, activation='softmax')(x1)
#preds = Dense(5, activation=act)(x1)
#
示例#33
0
def test_functional_guide():
    # MNIST
    from keras.layers import Input, Dense, LSTM
    from keras.models import Model
    from keras.utils import np_utils

    # this returns a tensor
    inputs = Input(shape=(784,))

    # a layer instance is callable on a tensor, and returns a tensor
    x = Dense(64, activation='relu')(inputs)
    x = Dense(64, activation='relu')(x)
    predictions = Dense(10, activation='softmax')(x)

    # this creates a model that includes
    # the Input layer and three Dense layers
    model = Model(input=inputs, output=predictions)
    model.compile(optimizer='rmsprop',
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    # the data, shuffled and split between tran and test sets
    X_train = np.random.random((100, 784))
    Y_train = np.random.random((100, 10))

    model.fit(X_train, Y_train, nb_epoch=2, batch_size=128)

    assert model.inputs == [inputs]
    assert model.outputs == [predictions]
    assert model.input == inputs
    assert model.output == predictions
    assert model.input_shape == (None, 784)
    assert model.output_shape == (None, 10)

    # try calling the sequential model
    inputs = Input(shape=(784,))
    new_outputs = model(inputs)
    new_model = Model(input=inputs, output=new_outputs)
    new_model.compile(optimizer='rmsprop',
                      loss='categorical_crossentropy',
                      metrics=['accuracy'])

    ##################################################
    # multi-io
    ##################################################
    tweet_a = Input(shape=(4, 25))
    tweet_b = Input(shape=(4, 25))
    # this layer can take as input a matrix
    # and will return a vector of size 64
    shared_lstm = LSTM(64)

    # when we reuse the same layer instance
    # multiple times, the weights of the layer
    # are also being reused
    # (it is effectively *the same* layer)
    encoded_a = shared_lstm(tweet_a)
    encoded_b = shared_lstm(tweet_b)

    # we can then concatenate the two vectors:
    merged_vector = merge([encoded_a, encoded_b],
                          mode='concat', concat_axis=-1)

    # and add a logistic regression on top
    predictions = Dense(1, activation='sigmoid')(merged_vector)

    # we define a trainable model linking the
    # tweet inputs to the predictions
    model = Model(input=[tweet_a, tweet_b], output=predictions)

    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy',
                  metrics=['accuracy'])
    data_a = np.random.random((1000, 4, 25))
    data_b = np.random.random((1000, 4, 25))
    labels = np.random.random((1000,))
    model.fit([data_a, data_b], labels, nb_epoch=1)

    model.summary()
    assert model.inputs == [tweet_a, tweet_b]
    assert model.outputs == [predictions]
    assert model.input == [tweet_a, tweet_b]
    assert model.output == predictions

    assert model.output == predictions
    assert model.input_shape == [(None, 4, 25), (None, 4, 25)]
    assert model.output_shape == (None, 1)

    assert shared_lstm.get_output_at(0) == encoded_a
    assert shared_lstm.get_output_at(1) == encoded_b
    assert shared_lstm.input_shape == (None, 4, 25)
def get_model_lstm_conv2d(input_profile_names, target_profile_names, scalar_input_names,
                          actuator_names, lookbacks, lookahead, profile_length, std_activation, **kwargs):

    profile_inshape = (profile_lookback, profile_length)
    actuator_inshape = (actuator_lookback + lookahead,)
    num_profiles = len(input_profile_names)
    num_targets = len(target_profile_names)
    num_actuators = len(actuator_names)
    max_channels = 32

    profile_inputs = []
    profiles = []
    for i in range(num_profiles):
        profile_inputs.append(
            Input(profile_inshape, name='input_' + input_profile_names[i]))
        profiles.append(Reshape((profile_lookback, profile_length, 1))
                        (profile_inputs[i]))
    profiles = Concatenate(axis=-1)(profiles)
    # shape = (lookback, length, channels=num_profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels/8), kernel_size=(1, 5),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels/4), kernel_size=(1, 10),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, 15),
                      strides=(1, 1), padding='same', activation=std_activation)(profiles)
    # shape = (lookback, length, channels)
    if profile_lookback > 1:
        profiles = Reshape((profile_lookback, 1, profile_length,
                            int(num_profiles*max_channels)))(profiles)
        # shape = (lookback, 1,  length, channels)
        profiles = ConvLSTM2D(filters=int(num_profiles*max_channels), kernel_size=(10, 1),
                              strides=(1, 1), padding='same', activation=std_activation,
                              recurrent_activation='hard_sigmoid')(profiles)
        #shape = (1, length, channels)
        profiles = Reshape((profile_length, int(
            num_profiles*max_channels)))(profiles)
        # shape = (length, channels)
    else:
        profiles = Conv2D(filters=int(num_profiles*max_channels), kernel_size=(1, 10),
                          strides=(1, 1), padding='same', activation=std_activation)(profiles)
        profiles = Reshape((profile_length, int(
            num_profiles*max_channels)))(profiles)
        # shape = (length, channels)

    actuator_inputs = []
    actuators = []
    for i in range(num_actuators):
        actuator_inputs.append(
            Input(actuator_inshape, name='input_' + actuator_names[i]))
        actuators.append(
            Reshape((actuator_lookback+lookahead, 1))(actuator_inputs[i]))
    actuators = Concatenate(axis=-1)(actuators)
    # shaoe = (time, num_actuators)
    actuators = Dense(units=int(num_profiles*max_channels/8),
                      activation=std_activation)(actuators)
    actuators = Conv1D(filters=int(num_profiles*max_channels/4), kernel_size=3, strides=1,
                       padding='causal', activation=std_activation)(actuators)
    actuators = LSTM(units=int(num_profiles*max_channels), activation=std_activation,
                     recurrent_activation='hard_sigmoid')(actuators)
    actuators = Reshape((int(num_profiles*max_channels), 1))(actuators)
    # shape = (channels, 1)
    actuators = Dense(units=profile_length,
                      activation=std_activation)(actuators)
    actuators = Dense(units=profile_length, activation=None)(actuators)
    # shape = (channels, profile_length)
    actuators = Permute(dims=(2, 1))(actuators)
    # shape = (profile_length, channels)

    merged = Add()([profiles, actuators])
    merged = Reshape((1, profile_length, int(
        num_profiles*max_channels)))(merged)
    # shape = (1, length, channels)

    prof_act = []
    for i in range(num_targets):
        prof_act.append(Conv2D(filters=max_channels, kernel_size=(1, 15), strides=(1, 1),
                               padding='same', activation=std_activation)(merged))
        # shape = (1,length,max_channels)
        prof_act[i] = Conv2D(filters=int(max_channels/4), kernel_size=(1, 15),
                             strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
        prof_act[i] = Conv2D(filters=int(max_channels/8), kernel_size=(1, 10),
                             strides=(1, 1), padding='same', activation=std_activation)(prof_act[i])
        prof_act[i] = Conv2D(filters=1, kernel_size=(1, 5), strides=(1, 1),
                             padding='same', activation=None)(prof_act[i])
        # shape = (1,length,1)
        prof_act[i] = Reshape((profile_length,), name='target_' +
                              target_profile_names[i])(prof_act[i])
    model = Model(inputs=profile_inputs + actuator_inputs, outputs=prof_act)
    return model
		# check if we are beyond the sequence
		if end_ix > len(sequence)-1:
			break
		# gather input and output parts of the pattern
		seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
		X.append(seq_x)
		y.append(seq_y)
	return array(X), array(y)

# define input sequence
raw_seq = [10, 20, 30, 40, 50, 60, 70, 80, 90]
# choose a number of time steps
n_steps = 3
# split into samples
X, y = split_sequence(raw_seq, n_steps)
# reshape from [samples, timesteps] into [samples, timesteps, features]
n_features = 1
X = X.reshape((X.shape[0], X.shape[1], n_features))
# define model
model = Sequential()
model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, n_features)))
model.add(LSTM(50, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit(X, y, epochs=200, verbose=0)
# demonstrate prediction
x_input = array([70, 80, 90])
x_input = x_input.reshape((1, n_steps, n_features))
yhat = model.predict(x_input, verbose=0)
print(yhat)
示例#36
0
x2_1 = x2

# split
x1_train, x1_test, y1_train, y1_test = train_test_split(
                                                     x1, y1, train_size =0.8, random_state =30)
x2_train, x2_test, y2_train, y2_test = train_test_split(
                                                     x2, y2, train_size =0.8, random_state =30)

np.save('D:/Study/test/x1_data5.npy', arr = x1)
np.save('D:/Study/test/x2_data5.npy', arr = x2)
np.save('D:/Study/test/y_data5.npy', arr = y1)

#2. model
# 1
input1 = Input(shape = (5,1 ))
dense1 = LSTM(50, activation = 'elu')(input1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(100, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(150, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(200, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)
dense1 = Dense(250, activation = 'elu')(dense1)
dense1 = Dropout(0.2)(dense1)

# 2
input2 = Input(shape = (5, 5))
dense2 = LSTM(100, activation = 'elu')(input2)
dense2 = Dropout(0.2)(dense2)
dense2 = Dense(150, activation = 'elu')(dense2)
示例#37
0
     int(x_train.shape[1] / features), features])
x_test = x_test.reshape(
    [x_test.shape[0],
     int(x_test.shape[1] / features), features])

#importing keras model and layers to construct LSTM model
from keras.models import Sequential
from keras.layers import Dense, Flatten, LSTM, Dropout

#initializing regression model
regressor = Sequential()

#adding layer(s) to model
regressor.add(
    LSTM(units=50,
         return_sequences=True,
         input_shape=(x_train.shape[1], x_train.shape[2])))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=50, return_sequences=True))
regressor.add(Dropout(0.2))
regressor.add(LSTM(units=33, return_sequences=True))

regressor.add(Flatten())
regressor.add(Dense(units=1))

#compiling the model with  mean_absolute_percentage_error and adam optimizer
regressor.compile(optimizer='adam', loss='mean_absolute_percentage_error')

#fitting model with training sets and validation set
history = regressor.fit(x_train,
                        y_train,
示例#38
0
for i in range(0, n_chars - seq_length, 1):
    seq_in = raw_text[i:i + seq_length]
    seq_out = raw_text[i + seq_length]
    dataX.append([char_to_int[char] for char in seq_in])
    dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print "Total Patterns: ", n_patterns
# reshape X to be [samples, time steps, features]
X = numpy.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
# define the LSTM model
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
# load the network weights
filename = "weights-improvement-19-1.9435.hdf5"
model.load_weights(filename)
model.compile(loss='categorical_crossentropy', optimizer='adam')
# pick a random seed
start = numpy.random.randint(0, len(dataX) - 1)
pattern = dataX[start]
print "Seed:"
print "\"", ''.join([int_to_char[value] for value in pattern]), "\""
# generate characters
for i in range(1000):
    x = numpy.reshape(pattern, (1, len(pattern), 1))
    x = x / float(n_vocab)