Пример #1
0
 def create(self):
     return models.Sequential([
         Model.Input((100, 41)),
         layers.LSTM(128, return_sequences=True, unroll=True, use_bias=True, name='lstm01'),
         layers.LSTM(128, return_sequences=False, unroll=True, use_bias=True, name='lstm02'),
         layers.Dense(41, activation=None, name='dense03')
     ], name=self.name)
Пример #2
0
def train(training_data,
          targets,
          epochs,
          batch_size,
          validation_split,
          hidden_neurons=5):
    # Resize data matrix to have 3 dimensions
    training_data = training_data[:, numpy.newaxis, :]
    targets = targets.transpose()

    training_model = models.Sequential()
    training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
    training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
    training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
    training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
    training_model.add(layers.LSTM(hidden_neurons))
    training_model.add(layers.Dense(1, activation='sigmoid'))

    training_model.compile(loss='binary_crossentropy',
                           optimizer='Adam',
                           metrics=['accuracy'])
    training_history = training_model.fit(training_data,
                                          targets,
                                          epochs=epochs,
                                          batch_size=batch_size,
                                          validation_split=validation_split)

    return training_model, training_history
Пример #3
0
def create_rnn_model(input_shape):
    inputs = layers.Input(shape=input_shape)
    x = layers.LSTM(128)(inputs)
    x = layers.RepeatVector(DIGITS + 1)(x)
    x = layers.LSTM(128, return_sequences=True)(x)
    x = layers.TimeDistributed(layers.Dense(len(CHARS)))(x)
    out = layers.Activation('softmax')(x)
    return models.Model(inputs=inputs, outputs=out)
def build_rnn_model():
    sequences = layers.Input(shape=(MAX_LENGTH, ))
    embedded = layers.Embedding(MAX_FEATURES, 64)(sequences)
    x = layers.LSTM(128, return_sequences=True)(embedded)
    x = layers.LSTM(128)(x)
    x = layers.Dense(32, activation='relu')(x)
    x = layers.Dense(100, activation='relu')(x)
    predictions = layers.Dense(1, activation='sigmoid')(x)
    model = models.Model(inputs=sequences, outputs=predictions)
    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy',
                  metrics=['binary_accuracy'])
    return model
def create_mem_network():
    sentence = layers.Input(shape=(story_maxlen,), dtype=tf.int32)
    encoded_sentence = layers.Embedding(input_dim=vocab_size, output_dim=50)(sentence)
    encoded_sentence = layers.Dropout(0.3)(encoded_sentence)

    question = layers.Input(shape=(query_maxlen,), dtype=tf.int32)
    encoded_ques = layers.Embedding(input_dim=vocab_size, output_dim=50)(question)
    encoded_ques = layers.Dropout(0.3)(encoded_ques)
    encoded_ques = layers.LSTM(50)(encoded_ques)
    encoded_ques = layers.RepeatVector(story_maxlen)(encoded_ques)

    merged = layers.add([encoded_sentence, encoded_ques])
    merged = layers.LSTM(50)(merged)
    merged = layers.Dropout(0.3)(merged)
    preds = layers.Dense(vocab_size, activation=None)(merged)
    return models.Model(inputs=[sentence, question], outputs=preds)
def create_mem_network():
    input_story = layers.Input(shape=(story_maxlen, ))
    input_m_encoded = layers.Embedding(input_dim=vocab_size,
                                       output_dim=64)(input_story)
    input_m_encoded = layers.Dropout(0.3)(input_m_encoded)

    input_c_encoded = layers.Embedding(input_dim=vocab_size,
                                       output_dim=query_maxlen)(input_story)
    input_c_encoded = layers.Dropout(0.3)(input_c_encoded)

    input_ques = layers.Input(shape=(query_maxlen, ))
    ques_encoded = layers.Embedding(input_dim=vocab_size,
                                    output_dim=64,
                                    input_length=query_maxlen)(input_ques)
    ques_encoded = layers.Dropout(0.3)(ques_encoded)
    # (samples, story_maxlen, query_maxlen)
    match = layers.dot([input_m_encoded, ques_encoded], axes=(2, 2))
    match = layers.Activation('softmax')(match)

    response = layers.add([match, input_c_encoded
                           ])  # (samples, story_maxlen, query_maxlen)
    response = layers.Permute((2, 1))(response)

    answer = layers.concatenate([response, ques_encoded])
    answer = layers.LSTM(32)(answer)
    answer = layers.Dropout(0.3)(answer)
    answer = layers.Dense(vocab_size, activation=None)(answer)

    return models.Model(inputs=[input_story, input_ques], outputs=answer)
Пример #7
0
def main():
    vocabulary_size = 10000
    maxlen = 24

    model = Sequential()
    model.add(layers.Embedding(vocabulary_size, 64, name="text"))
    model.add(
        layers.Conv1D(64, 4, padding='valid', activation='relu', strides=1))
    model.add(layers.MaxPooling1D(pool_size=3))
    model.add(layers.LSTM(64))
    model.add(layers.Dense(32, activation='relu'))
    model.add(layers.Dense(1, activation='sigmoid'))

    # # if use keras not tf.keras
    # model = tf.keras.models.Model(model)

    model.compile(optimizer='rmsprop',
                  loss='binary_crossentropy',
                  metrics=['acc'])

    estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model)

    data, labels = mr_load_data(max_word_num=vocabulary_size)
    data = pad_sequences(data, padding="pre", maxlen=maxlen)
    labels = np.asarray(labels).reshape(-1, 1)
    print(labels.shape)

    x_train, y_train = data, labels
    input_dict = {"text_input": x_train}
    input_fn = train_input_fn(input_dict, y_train, batch_size=32)
    print(input_fn)
    #
    # estimator_model.train(input_fn=input_fn, steps=10000)
    estimator_model.train(input_fn=input_function(input_dict, y_train),
                          steps=10000)
Пример #8
0
def create_lstm():
    model = tf.keras.Sequential()
    model.add(layers.Embedding(MAX_WORDS, 64, input_length=MAX_LEN))
    model.add(layers.Conv1D(32, 3, padding='same', activation='relu'))
    model.add(layers.MaxPooling1D(pool_size=4))
    model.add(layers.LSTM(64))
    model.add(layers.Dense(250, activation='relu'))
    model.add(layers.Dense(1, activation="sigmoid"))
    return model
Пример #9
0
def create_dialog_network(time_steps, classes_num):
    vocab_size = 60
    vec_size = 20
    model = Sequential()
    model.add(layers.Embedding(vocab_size, vec_size, input_length=time_steps))
    model.add(layers.LSTM(vec_size, dropout=0.2, recurrent_dropout=0.2))
    model.add(layers.Dense(classes_num, activation='softmax'))

    return model
Пример #10
0
def build_lstm(num_filters, vocab_size, dropout, embedding_dim, maxlen,
               optimizer):
    sequential_lstm = Sequential()
    sequential_lstm.add(
        layers.Embedding(input_dim=vocab_size,
                         output_dim=embedding_dim,
                         input_length=maxlen))
    sequential_lstm.add(layers.SpatialDropout1D(dropout))
    sequential_lstm.add(
        layers.LSTM(num_filters,
                    dropout=dropout,
                    recurrent_dropout=dropout,
                    return_sequences=True))
    sequential_lstm.add(
        layers.LSTM(num_filters, dropout=dropout, recurrent_dropout=dropout))
    sequential_lstm.add(layers.Dense(6, activation='softmax'))
    sequential_lstm.compile(loss='categorical_crossentropy',
                            optimizer=optimizer,
                            metrics=['accuracy'])
    return sequential_lstm
Пример #11
0
    def __init__(self, use_cudnn_lstm=True, plot_model_architecture=False):
        n_hidden = 50
        input_dim = 300

        # unit_forget_bias: Boolean. If True, add 1 to the bias of the forget gate at initialization. Setting it to true will also force  bias_initializer="zeros". This is recommended in Jozefowicz et al.
        # he_normal: Gaussian initialization scaled by fan_in (He et al., 2014)
        if use_cudnn_lstm:
            # Use CuDNNLSTM instead of LSTM, because it is faster
            lstm = layers.CuDNNLSTM(n_hidden,
                                    unit_forget_bias=True,
                                    kernel_initializer='he_normal',
                                    kernel_regularizer='l2',
                                    name='lstm_layer')
        else:
            lstm = layers.LSTM(n_hidden,
                               unit_forget_bias=True,
                               kernel_initializer='he_normal',
                               kernel_regularizer='l2',
                               name='lstm_layer')

        # Building the left branch of the model: inputs are variable-length sequences of vectors of size 128.
        left_input = Input(shape=(None, input_dim), name='input_1')
        #        left_masked_input = layers.Masking(mask_value=0)(left_input)
        left_output = lstm(left_input)

        # Building the right branch of the model: when you call an existing layer instance, you reuse its weights.
        right_input = Input(shape=(None, input_dim), name='input_2')
        #        right_masked_input = layers.Masking(mask_value=0)(right_input)
        right_output = lstm(right_input)

        # Builds the classifier on top
        l1_norm = lambda x: 1 - K.abs(x[0] - x[1])
        merged = layers.Lambda(function=l1_norm,
                               output_shape=lambda x: x[0],
                               name='L1_distance')([left_output, right_output])
        predictions = layers.Dense(1,
                                   activation='tanh',
                                   name='Similarity_layer')(merged)  #sigmoid

        # Instantiating and training the model: when you train such a model, the weights of the LSTM layer are updated based on both inputs.
        self.model = Model([left_input, right_input], predictions)

        self.__compile()
        print(self.model.summary())

        if plot_model_architecture:
            from tensorflow.python.keras.utils import plot_model
            plot_model(self.model, to_file='siamese_architecture.png')
    def testKerasModelWithRNNHealthyPredictAndFitCalls(self):
        """Test a simple healthy keras recurrent model works under the callback."""
        check_numerics_callback.enable_check_numerics()

        model = models.Sequential()
        model.add(layers.LSTM(1, input_shape=(2, 4)))
        model.compile(loss="mse", optimizer="rmsprop")

        xs = np.zeros([8, 2, 4], dtype=np.float32)
        ys = np.zeros([8, 1], dtype=np.float32)

        model.predict(xs)

        epochs = 3
        history = model.fit(xs, ys, epochs=epochs, verbose=0)
        self.assertEqual(len(history.history["loss"]), epochs)
Пример #13
0
  def testReturnsProvider_KerasRNNLayer(self):
    model = keras.Sequential([(
        l.LSTM(2, input_shape=(3, 2)))])
    layer = model.layers[0]

    quantize_provider = self.quantize_registry.get_quantize_provider(layer)

    (weights, weight_quantizers) = self._convert_list(
        quantize_provider.get_weights_and_quantizers(layer))
    (activations, activation_quantizers) = self._convert_list(
        quantize_provider.get_activations_and_quantizers(layer))

    self._assert_weight_quantizers(weight_quantizers)
    self.assertEqual([layer.cell.kernel, layer.cell.recurrent_kernel], weights)

    self._assert_activation_quantizers(activation_quantizers)
    self.assertEqual(
        [layer.cell.activation, layer.cell.recurrent_activation], activations)
Пример #14
0
    def __init__(self, state_size, nlayers=4, action_size=4):
        super(ActorCriticModel, self).__init__()

        self.state_size = state_size
        self.action_size = action_size
        self.nhidden = 1024

        self.shared = []
        #input_layer = layers.Dense( self.nhidden , activation = "relu" , input_shape = (4,4 , ) , kernel_initializer = "lecun_normal" )
        #input_layer = layers.Reshape( (4,4 , 1 ))
        #self.shared.append ( input_layer )
        for i in range(nlayers):

            #ayer = layers.Dense( self.nhidden , activation = "relu" , kernel_initializer = "lecun_normal" )
            #layer = layers.TimeDistributed ( layers.Conv2D( 32 , kernel_size = (1,1) , strides = (1,1) , padding = "VALID" ) )

            layer = layers.Conv2D(16,
                                  kernel_size=(2, 2),
                                  strides=(1, 1),
                                  activation="relu",
                                  kernel_initializer="lecun_normal",
                                  padding="same")
            self.shared.append(layer)
            layer = layers.Conv2DTranspose(16,
                                           kernel_size=(2, 2),
                                           strides=(1, 1),
                                           activation="relu",
                                           kernel_initializer="lecun_normal",
                                           padding="same")
            self.shared.append(layer)

            #self.layers_policy.append( do )

        self.shared.append(layers.TimeDistributed(layers.Flatten()))
        self.shared.append(layers.LSTM(self.nhidden))
        self.shared.append(layers.Flatten())
        #self.shared.append( layers.Dropout( 0.2 )  )

        self.logits = layers.Dense(4,
                                   activation="linear",
                                   kernel_initializer="lecun_normal")
        self.values = layers.Dense(1,
                                   activation="linear",
                                   kernel_initializer="lecun_normal")
Пример #15
0
def build_model(vocab_size,
                largest_vector_len,
                emb_dim=64,
                lstm_units=128,
                lr=1e-4,
                dropout_rate=0.5):
    """
    1D convolution and LSTM coming soon
    """
    tf.logging.set_verbosity(tf.logging.ERROR)

    # model = tf.keras.Sequential()

    # # Embedding layer
    # model.add(layers.Embedding(vocab_size, emb_dim, input_length=largest_vector_len))
    # model.add(layers.Dropout(dropout_rate))

    # # Convolutional layer
    # model.add(layers.Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
    # model.add(layers.MaxPooling1D(pool_size=2))

    # # LSTM layer
    # model.add(layers.LSTM(lstm_units, kernel_regularizer=regularizers.l2(0.05), activity_regularizer=regularizers.l1(0.05))) # , recurrent_dropout=dropout_rate)) # input_shape=(largest_vector_len, 4)))
    # # model.add(layers.Dropout(dropout_rate))

    # # TODO: Try some of the examples here: https://keras.io/getting-started/sequential-model-guide/

    # # Output layer
    # model.add(layers.Dense(1, activation='sigmoid'))

    model = tf.keras.Sequential()
    model.add(layers.Embedding(1024, output_dim=256))
    model.add(layers.LSTM(128))
    model.add(layers.Dropout(0.5))
    model.add(layers.Dense(1, activation='sigmoid'))

    optimizer = keras.optimizers.Adam(lr=lr)
    model.compile(loss='binary_crossentropy',
                  optimizer=optimizer,
                  metrics=['accuracy'])
    return model
Пример #16
0
 def __init__(self,
              units,
              use_bias=True,
              kernel_initializer='glorot_uniform',
              recurrent_initializer='orthogonal',
              bias_initializer='zeros',
              unit_forget_bias=True,
              dropout=0.,
              return_sequences=False,
              return_state=False,
              go_backwards=False,
              stateful=False,
              num_layers=1,
              bidirectional=False,
              **kwargs):
     super(LSTM, self).__init__(**kwargs)
     assert num_layers == 1, "Only support single layer for CuDNN RNN in keras"
     self._rnn = layers.LSTM(
         # cuDNN requirement
         activation='tanh',
         recurrent_activation='sigmoid',
         recurrent_dropout=0,
         unroll=False,
         use_bias=use_bias,
         # free arguments
         units=units,
         kernel_initializer=kernel_initializer,
         recurrent_initializer=recurrent_initializer,
         bias_initializer=bias_initializer,
         unit_forget_bias=unit_forget_bias,
         dropout=dropout,
         return_sequences=return_sequences,
         return_state=return_state,
         go_backwards=go_backwards,
         stateful=stateful,
         **kwargs)
     if bidirectional:
         self._rnn = layers.Bidirectional(
             self._rnn,
             merge_mode='concat',
         )
tower_2 = layers.Conv2D(32, (1, 1), padding='same')(layer_x)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
tower_2 = layers.Conv2D(32, (5, 1), padding='same')(tower_2)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)

tower_3 = layers.MaxPooling2D((3, 1), padding='same', strides=(1, 1))(layer_x)
tower_3 = layers.Conv2D(32, (1, 1), padding='same')(tower_3)
tower_3 = layers.LeakyReLU(alpha=0.01)(tower_3)

layer_x = layers.concatenate([tower_1, tower_2, tower_3], axis=-1)

# concatenate features of tower_1, tower_2, tower_3
layer_x = layers.Reshape((30, 96))(layer_x)

# 64 LSTM units
layer_x = layers.LSTM(256)(layer_x)
# The last output layer uses a softmax activation function
output = layers.Dense(3, activation='softmax')(layer_x)

model = Model(input_tensor, output)
opt = Adam(
    lr=0.01,
    epsilon=1)  # learning rate and epsilon are the same as paper DeepLOB
model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])
#print(model.summary())
'''conn = psycopg2.connect(**eval(open('auth.txt').read()))
cmd = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
#cmd.execute('select * from market_index where mid = 3 and dt=%(dt)s',dict(dt='2005-02-01'))
#recs = cmd.fetchall()
Пример #18
0
question_vocab_size = 10000
answer_vocab_size = 500

# Generating dummy data for demo
num_samples = 1000
max_len = 100
text = np.random.randint(1, text_vocab_size, size=(num_samples, max_len))
question = np.random.randint(1,
                             question_vocab_size,
                             size=(num_samples, max_len))
answers = np.random.randint(0, 2, size=(num_samples, answer_vocab_size))

text_input = Input(shape=(None, ), dtype='int32', name='text')
# embedded_text = layers.Embedding(64, text_vocab_size)(text_input)
embedded_text = layers.Embedding(text_vocab_size, 64)(text_input)
encoded_text = layers.LSTM(32)(embedded_text)

question_input = Input(shape=(None, ), dtype='int32', name='question')
# embedded_question = layers.Embedding(32, question_vocab_size)(question_input)
embedded_question = layers.Embedding(question_vocab_size, 32)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)

concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)
# print(concatenated.shape)       # 32 + 16 which gives us (?, 48) ... good
answer = layers.Dense(answer_vocab_size, activation='softmax')(concatenated)

model = Model([text_input, question_input], answer)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['acc'])
          Y_train,
          validation_data=(X_test, Y_test),
          callbacks=[es],
          epochs=50)

# # LSTM

# In[95]:

X_train, X_test = X_train.reshape(n_len, features,
                                  1), X_test.reshape(n_len_test, features, 1)

# In[97]:

model = Sequential(name="LSTM")
model.add(layers.LSTM(128, input_shape=(X_train.shape[1], 1)))
model.add(layers.Dense(6, activation="softmax"))

# In[98]:

model.compile(optimizer=optimizer,
              loss="categorical_crossentropy",
              metrics=["accuracy"])

# In[99]:

model.fit(X_train,
          Y_train,
          validation_data=(X_test, Y_test),
          callbacks=[es],
          epochs=50,
Пример #20
0
构建一个根据文档内容,标签和标题,预测文档优先级和执行部门的网络
超参
'''
num_words = 2000
num_tags = 12
num_departments = 4
'''输入'''
body_input = tf.keras.layers.Input(shape=(None, ), name='body')
title_input = tf.keras.layers.Input(shape=(None, ), name='title')
tag_input = tf.keras.layers.Input(shape=(num_tags, ), name='tag')
'''嵌入层'''
body_feat = layers.Embedding(num_words, 64)(body_input)
title_feat = layers.Embedding(num_words, 64)(title_input)
'''特征提取层'''

body_feat = layers.LSTM(32)(body_feat)
title_feat = layers.LSTM(32)(title_feat)
features = layers.concatenate([title_feat, body_feat, tag_input])
print(tag_input, body_feat, title_feat, features)
''' 分类层'''
priority_pred = layers.Dense(1, activation='sigmoid',
                             name='priority')(features)
department_pred = layers.Dense(num_departments,
                               activation='softmax',
                               name='department')(features)
print(priority_pred, department_pred)
'''构建模型'''
model = tf.keras.Model(inputs=[body_input, title_input, tag_input],
                       outputs=[priority_pred, department_pred])

model.summary()
Пример #21
0
Y = complain_data['Satisfaction'].values
X_train, X_test, Y_train, Y_test = train_test_split(
    X, Y, test_size=0.33, random_state=42)

# print(X_train.shape,Y_train.shape)
# print(X_test.shape,Y_test.shape)

embedding_vector_length = 32

model = Sequential()
model.add(layers.Embedding(max_features,
          embedding_vector_length, input_length=maxlen))
model.add(layers.Conv1D(filters=32, kernel_size=3,
          padding='same', activation='relu'))
model.add(layers.MaxPooling1D(pool_size=2))
model.add(layers.LSTM(100, recurrent_activation='sigmoid'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='adam', metrics=['accuracy'])
model.summary()

filename = os.path.join(current_dir, 'data', 'complain_model.h5')
is_training = False
if is_training:
    model.fit(X_train, Y_train, validation_data=(
        X_test, Y_test), epochs=20, batch_size=64)

    # Evaluate the model
    scores = model.evaluate(X_test, Y_test, verbose=0)
    print("Evaluation Accuracy: %.2f%%" % (scores[1]*100))
    model.save(filename, save_format='tf')
Пример #22
0
        # if using GPU
        if tf.test.is_gpu_available():
            print('using GPU')
            layer_x = layers.CuDNNLSTM(
                lstm_units,
                kernel_regularizer=regularizers.l1(l=regularizer),
                kernel_initializer="zeros",
                recurrent_initializer="zeros",
                bias_initializer="zeros")(layer_x)
        # if using CPU
        else:
            print('using CPU')
            layer_x = layers.LSTM(
                lstm_units,
                kernel_regularizer=regularizers.l1(l=regularizer),
                kernel_initializer="zeros",
                recurrent_initializer="zeros",
                bias_initializer="zeros")(layer_x)

        # # The last output layer uses a softmax activation function
        output = layers.Dense(3,
                              activation='softmax',
                              kernel_initializer="zeros",
                              bias_initializer="zeros")(layer_x)

        if feature_num == 4:
            model = Model(input_tensor, output)

        elif feature_num == 5:
            model = Model([input_tensor, input_tensor2], output)
Пример #23
0
          Y_train,
          validation_data=(X_test, Y_test),
          callbacks=[es],
          epochs=50)

# # LSTM

# In[ ]:

X_train, X_test = X_train.reshape(n_len, features,
                                  1), X_test.reshape(n_len_test, features, 1)

# In[ ]:

model = Sequential(name="LSTM")
model.add(layers.LSTM(128, input_shape=(X_train.shape[1], 1)))
model.add(layers.Dense(6, activation="softmax"))

# In[ ]:

model.summary()

# In[ ]:

model.compile(optimizer=optimizer,
              loss="categorical_crossentropy",
              metrics=["accuracy"])

# In[ ]:

model.fit(X_train,
Пример #24
0
X_train /= std_train

X_test -= mean_train
X_test /= std_train

# reshape to be suitable for Keras
y_train = utils.to_categorical(y_train)
y_test = utils.to_categorical(y_test)

print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)

model = Sequential()
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                input_shape=(None, num_channels * 30 * FreqSample / step),
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
Пример #25
0
 def testSupports_KerasRNNLayers(self):
   self.assertTrue(self.quantize_registry.supports(l.LSTM(10)))
   self.assertTrue(self.quantize_registry.supports(l.GRU(10)))
Пример #26
0
    val_loss = history.history['val_loss']
    train_acc = history.history['acc']
    val_acc = history.history['val_acc']
    print(train_loss)
    print(val_loss)
    print(train_acc)
    print(val_acc)


model = Sequential(name='rnn')
model.add(
    layers.BatchNormalization(input_shape=(None, num_channels * 30 *
                                           FreqSample / step)))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
                return_sequences=True))
model.add(
    layers.LSTM(1000,
                dropout=0.1,
                recurrent_dropout=0.1,
Пример #27
0
def main(thread_num):
    # causing allocating 2 gpu devices
    # if tf.test.is_gpu_available():

    # select gpu device 0,1
    if device_type == 'gpu':
        os.environ["CUDA_VISIBLE_DEVICES"] = thread_num

    task = pd.read_csv("task.csv")
    #print(task)

    if os.path.isfile("output.csv"):
        output_csv = pd.read_csv("output.csv")
    else:
        output_csv = task
        output_csv = output_csv.drop(columns=['data_set'])

        #output_csv['train_acc'] = 0.0
        output_csv['final_train_loss'] = 100.0
        #output_csv['valid_acc'] = 0.0
        output_csv['final_valid_loss'] = 100.0
        output_csv['best_trade_acc'] = 0.0
        output_csv['best_trade_acc_epoch'] = 0
        output_csv['best_trade_f1'] = 0.0
        output_csv['best_trade_f1_epoch'] = 0
        output_csv['best_trade_precision'] = 0.0
        output_csv['best_trade_precision_epoch'] = 0
        output_csv['best_trade_recall'] = 0.0
        output_csv['best_trade_recall_epoch'] = 0
        # output_csv['best_trade_loss'] = 100.0
        # output_csv['best_trade_loss_epoch'] = 0

        output_csv['completed'] = 0

    for index, row in task.iterrows():

        #if tf.test.is_gpu_available():
        if device_type == 'gpu':
            if index % 2 != int(thread_num):
                continue

        completed = output_csv['completed'][index]
        if completed == 1:
            continue

        data_set = int(task['data_set'][index])

        load_dir = os.path.join(os.getcwd(), 'data_set/' + str(data_set))
        if not os.path.isdir(load_dir):
            continue

        task_id = int(task['task_id'][index])
        input_size = int(task['input'][index])
        pred_k = int(task['k'][index])
        feature_num = int(task['feature_num'][index])
        label_threshold = float(task['label_threshold'][index])
        lstm_units = int(task['lstm_units'][index])
        lr = float(task['learning_rate'][index])
        epsilon = float(task['epsilon'][index])
        regularizer = float(task['regularizer'][index])

        train_x = np.load(os.path.join(load_dir, 'train_x.npy'))
        train_y = np.load(os.path.join(load_dir, 'train_y_onehot.npy'))
        valid_x = np.load(os.path.join(load_dir, 'valid_x.npy'))
        valid_y = np.load(os.path.join(load_dir, 'valid_y_onehot.npy'))
        trade_y = np.load(os.path.join(load_dir, 'trading_valid_y_onehot.npy'))

        print('Running experiment {}'.format(task_id))

        #clear previous models
        clear_session()

        model_dir = os.path.join(os.getcwd(), 'load_model')

        if os.path.isdir(model_dir):
            model_dir = os.path.join(
                model_dir,
                str(task_id) + '/model/model_epoch_500.h5')
            if not os.path.isdir(model_dir):
                continue

            model = load_model(model_dir)
        else:

            #input_tensor = Input(shape=(30,4,1))
            input_tensor = Input(shape=(input_size, 4, 1))

            layer_x = layers.Conv2D(16, (1, 4),
                                    kernel_regularizer=regularizers.l1(
                                        l=regularizer))(input_tensor)
            layer_x = layers.BatchNormalization()(layer_x)
            layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)

            layer_x = layers.Conv2D(
                16, (4, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            layer_x = layers.BatchNormalization()(layer_x)
            layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)

            layer_x = layers.Conv2D(
                16, (4, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            layer_x = layers.BatchNormalization()(layer_x)
            layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)

            #dual input for ohlc+volume
            if feature_num == 5:
                train_x_ohlc = train_x[:, :, :4, :]
                train_x_volume = train_x[:, :, -1:, :]
                train_x = [train_x_ohlc, train_x_volume]

                valid_x_ohlc = valid_x[:, :, :4, :]
                valid_x_volume = valid_x[:, :, -1:, :]
                valid_x = [valid_x_ohlc, valid_x_volume]

                input_tensor2 = Input(shape=(input_size, 1, 1))

                layer_x2 = layers.Conv2D(16, (1, 1),
                                         kernel_regularizer=regularizers.l1(
                                             l=regularizer))(input_tensor2)
                layer_x2 = layers.BatchNormalization()(layer_x2)
                layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)

                layer_x2 = layers.Conv2D(16, (4, 1),
                                         padding='same',
                                         kernel_regularizer=regularizers.l1(
                                             l=regularizer))(layer_x2)
                layer_x2 = layers.BatchNormalization()(layer_x2)
                layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)

                layer_x2 = layers.Conv2D(16, (4, 1),
                                         padding='same',
                                         kernel_regularizer=regularizers.l1(
                                             l=regularizer))(layer_x2)
                layer_x2 = layers.BatchNormalization()(layer_x2)
                layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)

                layer_x = layers.concatenate([layer_x, layer_x2], axis=-1)

            # Inception Module
            tower_1 = layers.Conv2D(
                32, (1, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            tower_1 = layers.BatchNormalization()(tower_1)
            tower_1 = layers.LeakyReLU(alpha=0.01)(tower_1)
            tower_1 = layers.Conv2D(
                32, (3, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(tower_1)
            tower_1 = layers.BatchNormalization()(tower_1)
            tower_1 = layers.LeakyReLU(alpha=0.01)(tower_1)

            tower_2 = layers.Conv2D(
                32, (1, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            tower_2 = layers.BatchNormalization()(tower_2)
            tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
            tower_2 = layers.Conv2D(
                32, (5, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(tower_2)
            tower_2 = layers.BatchNormalization()(tower_2)
            tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)

            tower_3 = layers.MaxPooling2D((3, 1),
                                          padding='same',
                                          strides=(1, 1))(layer_x)
            tower_3 = layers.Conv2D(
                32, (1, 1),
                padding='same',
                kernel_regularizer=regularizers.l1(l=regularizer))(tower_3)
            tower_3 = layers.BatchNormalization()(tower_3)
            tower_3 = layers.LeakyReLU(alpha=0.01)(tower_3)

            layer_x = layers.concatenate([tower_1, tower_2, tower_3], axis=-1)

            # concatenate features of tower_1, tower_2, tower_3
            layer_x = layers.Reshape((input_size, 96))(layer_x)
            #layer_x = layers.Reshape((input_size,feature_num))(input_tensor)

            # # 64 LSTM units
            #layer_x = layers.LSTM(64)(layer_x)

            # if using GPU
            if device_type == 'gpu':
                print('using GPU')
                layer_x = layers.CuDNNLSTM(
                    lstm_units,
                    kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            # if using CPU
            elif device_type == 'cpu':
                print('using CPU')
                layer_x = layers.LSTM(
                    lstm_units,
                    kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
            else:
                sys.exit("wrong device type")

            # # The last output layer uses a softmax activation function
            output = layers.Dense(3, activation='softmax')(layer_x)

            if feature_num == 4:
                model = Model(input_tensor, output)

            elif feature_num == 5:
                model = Model([input_tensor, input_tensor2], output)

            opt = Adam(lr=lr, epsilon=epsilon)
            model.compile(loss='categorical_crossentropy',
                          optimizer=opt,
                          metrics=['accuracy'])
            #model.summary()

        save_dir = os.path.join(os.getcwd(), 'result/' + str(task_id))
        if not os.path.isdir(save_dir):
            os.makedirs(save_dir)

        final_train_loss, \
        final_valid_loss, \
        best_trade_acc, \
        best_trade_acc_epoch, \
        best_trade_f1, \
        best_trade_f1_epoch, \
        best_trade_precision, \
        best_trade_precision_epoch, \
        best_trade_recall, \
        best_trade_recall_epoch \
         = train_model(model, \
                      save_dir, \
                      task_id, \
                      train_x, \
                      train_y, \
                      valid_x, \
                      valid_y, \
                      trade_y, \
                      batch_size=512, \
                      epochs=3)

        with open(os.path.join(save_dir, 'readme.txt'), 'w') as f:
            f.write("""'task id = {}\n
                        input size = {}\n
                        prediction k = {}\n
                        feature = {}\n
                        label threshold = {}\n
                        lstm units = {}\n
                        learning rate = {}\n
                        epsilon = {}\n
                        regularizer = {}\n
                        data set = {}'""".format(task_id, \
                                                input_size, \
                                                pred_k, \
                                                feature_num, \
                                                label_threshold, \
                                                lstm_units, \
                                                lr, \
                                                epsilon, \
                                                regularizer, \
                                                data_set))

        #output_csv['train_acc'][index] = train_acc
        output_csv['final_train_loss'][index] = final_train_loss
        #output_csv['valid_acc'][index] = valid_acc
        output_csv['final_valid_loss'][index] = final_valid_loss
        output_csv['best_trade_acc'][index] = best_trade_acc
        output_csv['best_trade_acc_epoch'][index] = best_trade_acc_epoch
        output_csv['best_trade_f1'][index] = best_trade_f1
        output_csv['best_trade_f1_epoch'][index] = best_trade_f1_epoch
        output_csv['best_trade_precision'][index] = best_trade_precision
        output_csv['best_trade_precision_epoch'][
            index] = best_trade_precision_epoch
        output_csv['best_trade_recall'][index] = best_trade_recall
        output_csv['best_trade_recall_epoch'][index] = best_trade_recall_epoch
        #output_csv['best_trade_loss'] = best_trade_loss
        #output_csv['best_trade_loss_epoch'] = best_trade_loss_epoch

        output_csv['completed'][index] = 1

        output_csv.to_csv('output.csv')
Пример #28
0
        #layer_x = layers.Reshape((input_size,feature_num))(input_tensor)

        # # 64 LSTM units
        #layer_x = layers.LSTM(64)(layer_x)

        # if using GPU
        if tf.test.is_gpu_available():
            print('using GPU')
            layer_x = layers.CuDNNLSTM(
                lstm_units,
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
        # if using CPU
        else:
            print('using CPU')
            layer_x = layers.LSTM(
                lstm_units,
                kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)

        # # The last output layer uses a softmax activation function
        output = layers.Dense(3, activation='softmax')(layer_x)

        if feature_num == 4:
            model = Model(input_tensor, output)

        elif feature_num == 5:
            model = Model([input_tensor, input_tensor2], output)

        opt = Adam(lr=lr, epsilon=1
                   )  # learning rate and epsilon are the same as paper DeepLOB
        model.compile(loss='categorical_crossentropy',
                      optimizer=opt,
Пример #29
0
xtrain = tokenizer.texts_to_sequences(x_train)
xtest = tokenizer.texts_to_sequences(x_test)

maxlen = 10
xtrain = pad_sequences(xtrain, padding='post', maxlen=maxlen)
xtest = pad_sequences(xtest, padding='post', maxlen=maxlen)

vocab_size = len(tokenizer.word_index) + 1

embedding_dim = 50
model = Sequential()
model.add(
    layers.Embedding(input_dim=vocab_size,
                     output_dim=embedding_dim,
                     input_length=maxlen))
model.add(layers.LSTM(units=50, return_sequences=True))
model.add(layers.LSTM(units=10))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(8))
model.add(layers.Dense(1, activation="sigmoid"))
model.compile(optimizer="adam",
              loss="binary_crossentropy",
              metrics=['accuracy'])
model.summary()

model.fit(xtrain, y_train, epochs=20, batch_size=16, verbose=True)
model.save('my_model')
print('model saved!')

loss, acc = model.evaluate(xtrain, y_train, verbose=True)
print("Training Accuracy: ", acc)
from tensorflow.python.keras import Input
from tensorflow.python.keras import Model
from tensorflow.python.keras import layers
import tensorflow as tf

main_input = Input(shape=(100, ), dtype='int32', name='main_input')
x = layers.Embedding(output_dim=512, input_dim=10000,
                     input_length=100)(main_input)
lstm_out = layers.LSTM(32)(x)
auxiliary_output = layers.Dense(1, activation='sigmoid',
                                name='aux_output')(lstm_out)
aux_input = Input(shape=(5, ), name='aux_input')
x = layers.concatenate([lstm_out, aux_input])
x = layers.Dense(64, activation='relu')(x)
x = layers.Dense(64, activation='relu')(x)
x = layers.Dense(64, activation='relu')(x)
main_output = layers.Dense(1, activation='sigmoid', name='main_output')(x)

model = Model(inputs=[main_input, aux_input],
              outputs=[main_output, auxiliary_output])
model.compile(optimizer='rmsprop',
              loss='binary_crossentropy',
              loss_weights=[1., 0.2])