def create(self):
return models.Sequential([
Model.Input((100, 41)),
layers.LSTM(128, return_sequences=True, unroll=True, use_bias=True, name='lstm01'),
layers.LSTM(128, return_sequences=False, unroll=True, use_bias=True, name='lstm02'),
layers.Dense(41, activation=None, name='dense03')
], name=self.name)
def train(training_data,
targets,
epochs,
batch_size,
validation_split,
hidden_neurons=5):
# Resize data matrix to have 3 dimensions
training_data = training_data[:, numpy.newaxis, :]
targets = targets.transpose()
training_model = models.Sequential()
training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
training_model.add(layers.LSTM(hidden_neurons, return_sequences=True))
training_model.add(layers.LSTM(hidden_neurons))
training_model.add(layers.Dense(1, activation='sigmoid'))
training_model.compile(loss='binary_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
training_history = training_model.fit(training_data,
targets,
epochs=epochs,
batch_size=batch_size,
validation_split=validation_split)
return training_model, training_history
def create_rnn_model(input_shape):
inputs = layers.Input(shape=input_shape)
x = layers.LSTM(128)(inputs)
x = layers.RepeatVector(DIGITS + 1)(x)
x = layers.LSTM(128, return_sequences=True)(x)
x = layers.TimeDistributed(layers.Dense(len(CHARS)))(x)
out = layers.Activation('softmax')(x)
return models.Model(inputs=inputs, outputs=out)
def build_rnn_model():
sequences = layers.Input(shape=(MAX_LENGTH, ))
embedded = layers.Embedding(MAX_FEATURES, 64)(sequences)
x = layers.LSTM(128, return_sequences=True)(embedded)
x = layers.LSTM(128)(x)
x = layers.Dense(32, activation='relu')(x)
x = layers.Dense(100, activation='relu')(x)
predictions = layers.Dense(1, activation='sigmoid')(x)
model = models.Model(inputs=sequences, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['binary_accuracy'])
return model
def create_mem_network():
sentence = layers.Input(shape=(story_maxlen,), dtype=tf.int32)
encoded_sentence = layers.Embedding(input_dim=vocab_size, output_dim=50)(sentence)
encoded_sentence = layers.Dropout(0.3)(encoded_sentence)
question = layers.Input(shape=(query_maxlen,), dtype=tf.int32)
encoded_ques = layers.Embedding(input_dim=vocab_size, output_dim=50)(question)
encoded_ques = layers.Dropout(0.3)(encoded_ques)
encoded_ques = layers.LSTM(50)(encoded_ques)
encoded_ques = layers.RepeatVector(story_maxlen)(encoded_ques)
merged = layers.add([encoded_sentence, encoded_ques])
merged = layers.LSTM(50)(merged)
merged = layers.Dropout(0.3)(merged)
preds = layers.Dense(vocab_size, activation=None)(merged)
return models.Model(inputs=[sentence, question], outputs=preds)
def create_mem_network():
input_story = layers.Input(shape=(story_maxlen, ))
input_m_encoded = layers.Embedding(input_dim=vocab_size,
output_dim=64)(input_story)
input_m_encoded = layers.Dropout(0.3)(input_m_encoded)
input_c_encoded = layers.Embedding(input_dim=vocab_size,
output_dim=query_maxlen)(input_story)
input_c_encoded = layers.Dropout(0.3)(input_c_encoded)
input_ques = layers.Input(shape=(query_maxlen, ))
ques_encoded = layers.Embedding(input_dim=vocab_size,
output_dim=64,
input_length=query_maxlen)(input_ques)
ques_encoded = layers.Dropout(0.3)(ques_encoded)
# (samples, story_maxlen, query_maxlen)
match = layers.dot([input_m_encoded, ques_encoded], axes=(2, 2))
match = layers.Activation('softmax')(match)
response = layers.add([match, input_c_encoded
]) # (samples, story_maxlen, query_maxlen)
response = layers.Permute((2, 1))(response)
answer = layers.concatenate([response, ques_encoded])
answer = layers.LSTM(32)(answer)
answer = layers.Dropout(0.3)(answer)
answer = layers.Dense(vocab_size, activation=None)(answer)
return models.Model(inputs=[input_story, input_ques], outputs=answer)
def main():
vocabulary_size = 10000
maxlen = 24
model = Sequential()
model.add(layers.Embedding(vocabulary_size, 64, name="text"))
model.add(
layers.Conv1D(64, 4, padding='valid', activation='relu', strides=1))
model.add(layers.MaxPooling1D(pool_size=3))
model.add(layers.LSTM(64))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# # if use keras not tf.keras
# model = tf.keras.models.Model(model)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc'])
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model)
data, labels = mr_load_data(max_word_num=vocabulary_size)
data = pad_sequences(data, padding="pre", maxlen=maxlen)
labels = np.asarray(labels).reshape(-1, 1)
print(labels.shape)
x_train, y_train = data, labels
input_dict = {"text_input": x_train}
input_fn = train_input_fn(input_dict, y_train, batch_size=32)
print(input_fn)
#
# estimator_model.train(input_fn=input_fn, steps=10000)
estimator_model.train(input_fn=input_function(input_dict, y_train),
steps=10000)
def create_lstm():
model = tf.keras.Sequential()
model.add(layers.Embedding(MAX_WORDS, 64, input_length=MAX_LEN))
model.add(layers.Conv1D(32, 3, padding='same', activation='relu'))
model.add(layers.MaxPooling1D(pool_size=4))
model.add(layers.LSTM(64))
model.add(layers.Dense(250, activation='relu'))
model.add(layers.Dense(1, activation="sigmoid"))
return model
def create_dialog_network(time_steps, classes_num):
vocab_size = 60
vec_size = 20
model = Sequential()
model.add(layers.Embedding(vocab_size, vec_size, input_length=time_steps))
model.add(layers.LSTM(vec_size, dropout=0.2, recurrent_dropout=0.2))
model.add(layers.Dense(classes_num, activation='softmax'))
return model
def build_lstm(num_filters, vocab_size, dropout, embedding_dim, maxlen,
optimizer):
sequential_lstm = Sequential()
sequential_lstm.add(
layers.Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=maxlen))
sequential_lstm.add(layers.SpatialDropout1D(dropout))
sequential_lstm.add(
layers.LSTM(num_filters,
dropout=dropout,
recurrent_dropout=dropout,
return_sequences=True))
sequential_lstm.add(
layers.LSTM(num_filters, dropout=dropout, recurrent_dropout=dropout))
sequential_lstm.add(layers.Dense(6, activation='softmax'))
sequential_lstm.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return sequential_lstm
def __init__(self, use_cudnn_lstm=True, plot_model_architecture=False):
n_hidden = 50
input_dim = 300
# unit_forget_bias: Boolean. If True, add 1 to the bias of the forget gate at initialization. Setting it to true will also force bias_initializer="zeros". This is recommended in Jozefowicz et al.
# he_normal: Gaussian initialization scaled by fan_in (He et al., 2014)
if use_cudnn_lstm:
# Use CuDNNLSTM instead of LSTM, because it is faster
lstm = layers.CuDNNLSTM(n_hidden,
unit_forget_bias=True,
kernel_initializer='he_normal',
kernel_regularizer='l2',
name='lstm_layer')
else:
lstm = layers.LSTM(n_hidden,
unit_forget_bias=True,
kernel_initializer='he_normal',
kernel_regularizer='l2',
name='lstm_layer')
# Building the left branch of the model: inputs are variable-length sequences of vectors of size 128.
left_input = Input(shape=(None, input_dim), name='input_1')
# left_masked_input = layers.Masking(mask_value=0)(left_input)
left_output = lstm(left_input)
# Building the right branch of the model: when you call an existing layer instance, you reuse its weights.
right_input = Input(shape=(None, input_dim), name='input_2')
# right_masked_input = layers.Masking(mask_value=0)(right_input)
right_output = lstm(right_input)
# Builds the classifier on top
l1_norm = lambda x: 1 - K.abs(x[0] - x[1])
merged = layers.Lambda(function=l1_norm,
output_shape=lambda x: x[0],
name='L1_distance')([left_output, right_output])
predictions = layers.Dense(1,
activation='tanh',
name='Similarity_layer')(merged) #sigmoid
# Instantiating and training the model: when you train such a model, the weights of the LSTM layer are updated based on both inputs.
self.model = Model([left_input, right_input], predictions)
self.__compile()
print(self.model.summary())
if plot_model_architecture:
from tensorflow.python.keras.utils import plot_model
plot_model(self.model, to_file='siamese_architecture.png')
def testKerasModelWithRNNHealthyPredictAndFitCalls(self):
"""Test a simple healthy keras recurrent model works under the callback."""
check_numerics_callback.enable_check_numerics()
model = models.Sequential()
model.add(layers.LSTM(1, input_shape=(2, 4)))
model.compile(loss="mse", optimizer="rmsprop")
xs = np.zeros([8, 2, 4], dtype=np.float32)
ys = np.zeros([8, 1], dtype=np.float32)
model.predict(xs)
epochs = 3
history = model.fit(xs, ys, epochs=epochs, verbose=0)
self.assertEqual(len(history.history["loss"]), epochs)
def testReturnsProvider_KerasRNNLayer(self):
model = keras.Sequential([(
l.LSTM(2, input_shape=(3, 2)))])
layer = model.layers[0]
quantize_provider = self.quantize_registry.get_quantize_provider(layer)
(weights, weight_quantizers) = self._convert_list(
quantize_provider.get_weights_and_quantizers(layer))
(activations, activation_quantizers) = self._convert_list(
quantize_provider.get_activations_and_quantizers(layer))
self._assert_weight_quantizers(weight_quantizers)
self.assertEqual([layer.cell.kernel, layer.cell.recurrent_kernel], weights)
self._assert_activation_quantizers(activation_quantizers)
self.assertEqual(
[layer.cell.activation, layer.cell.recurrent_activation], activations)
def __init__(self, state_size, nlayers=4, action_size=4):
super(ActorCriticModel, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.nhidden = 1024
self.shared = []
#input_layer = layers.Dense( self.nhidden , activation = "relu" , input_shape = (4,4 , ) , kernel_initializer = "lecun_normal" )
#input_layer = layers.Reshape( (4,4 , 1 ))
#self.shared.append ( input_layer )
for i in range(nlayers):
#ayer = layers.Dense( self.nhidden , activation = "relu" , kernel_initializer = "lecun_normal" )
#layer = layers.TimeDistributed ( layers.Conv2D( 32 , kernel_size = (1,1) , strides = (1,1) , padding = "VALID" ) )
layer = layers.Conv2D(16,
kernel_size=(2, 2),
strides=(1, 1),
activation="relu",
kernel_initializer="lecun_normal",
padding="same")
self.shared.append(layer)
layer = layers.Conv2DTranspose(16,
kernel_size=(2, 2),
strides=(1, 1),
activation="relu",
kernel_initializer="lecun_normal",
padding="same")
self.shared.append(layer)
#self.layers_policy.append( do )
self.shared.append(layers.TimeDistributed(layers.Flatten()))
self.shared.append(layers.LSTM(self.nhidden))
self.shared.append(layers.Flatten())
#self.shared.append( layers.Dropout( 0.2 ) )
self.logits = layers.Dense(4,
activation="linear",
kernel_initializer="lecun_normal")
self.values = layers.Dense(1,
activation="linear",
kernel_initializer="lecun_normal")
def build_model(vocab_size,
largest_vector_len,
emb_dim=64,
lstm_units=128,
lr=1e-4,
dropout_rate=0.5):
"""
1D convolution and LSTM coming soon
"""
tf.logging.set_verbosity(tf.logging.ERROR)
# model = tf.keras.Sequential()
# # Embedding layer
# model.add(layers.Embedding(vocab_size, emb_dim, input_length=largest_vector_len))
# model.add(layers.Dropout(dropout_rate))
# # Convolutional layer
# model.add(layers.Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
# model.add(layers.MaxPooling1D(pool_size=2))
# # LSTM layer
# model.add(layers.LSTM(lstm_units, kernel_regularizer=regularizers.l2(0.05), activity_regularizer=regularizers.l1(0.05))) # , recurrent_dropout=dropout_rate)) # input_shape=(largest_vector_len, 4)))
# # model.add(layers.Dropout(dropout_rate))
# # TODO: Try some of the examples here: https://keras.io/getting-started/sequential-model-guide/
# # Output layer
# model.add(layers.Dense(1, activation='sigmoid'))
model = tf.keras.Sequential()
model.add(layers.Embedding(1024, output_dim=256))
model.add(layers.LSTM(128))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))
optimizer = keras.optimizers.Adam(lr=lr)
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
return model
def __init__(self,
units,
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
dropout=0.,
return_sequences=False,
return_state=False,
go_backwards=False,
stateful=False,
num_layers=1,
bidirectional=False,
**kwargs):
super(LSTM, self).__init__(**kwargs)
assert num_layers == 1, "Only support single layer for CuDNN RNN in keras"
self._rnn = layers.LSTM(
# cuDNN requirement
activation='tanh',
recurrent_activation='sigmoid',
recurrent_dropout=0,
unroll=False,
use_bias=use_bias,
# free arguments
units=units,
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
unit_forget_bias=unit_forget_bias,
dropout=dropout,
return_sequences=return_sequences,
return_state=return_state,
go_backwards=go_backwards,
stateful=stateful,
**kwargs)
if bidirectional:
self._rnn = layers.Bidirectional(
self._rnn,
merge_mode='concat',
)
tower_2 = layers.Conv2D(32, (1, 1), padding='same')(layer_x)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
tower_2 = layers.Conv2D(32, (5, 1), padding='same')(tower_2)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
tower_3 = layers.MaxPooling2D((3, 1), padding='same', strides=(1, 1))(layer_x)
tower_3 = layers.Conv2D(32, (1, 1), padding='same')(tower_3)
tower_3 = layers.LeakyReLU(alpha=0.01)(tower_3)
layer_x = layers.concatenate([tower_1, tower_2, tower_3], axis=-1)
# concatenate features of tower_1, tower_2, tower_3
layer_x = layers.Reshape((30, 96))(layer_x)
# 64 LSTM units
layer_x = layers.LSTM(256)(layer_x)
# The last output layer uses a softmax activation function
output = layers.Dense(3, activation='softmax')(layer_x)
model = Model(input_tensor, output)
opt = Adam(
lr=0.01,
epsilon=1) # learning rate and epsilon are the same as paper DeepLOB
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#print(model.summary())
'''conn = psycopg2.connect(**eval(open('auth.txt').read()))
cmd = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
#cmd.execute('select * from market_index where mid = 3 and dt=%(dt)s',dict(dt='2005-02-01'))
#recs = cmd.fetchall()
question_vocab_size = 10000
answer_vocab_size = 500
# Generating dummy data for demo
num_samples = 1000
max_len = 100
text = np.random.randint(1, text_vocab_size, size=(num_samples, max_len))
question = np.random.randint(1,
question_vocab_size,
size=(num_samples, max_len))
answers = np.random.randint(0, 2, size=(num_samples, answer_vocab_size))
text_input = Input(shape=(None, ), dtype='int32', name='text')
# embedded_text = layers.Embedding(64, text_vocab_size)(text_input)
embedded_text = layers.Embedding(text_vocab_size, 64)(text_input)
encoded_text = layers.LSTM(32)(embedded_text)
question_input = Input(shape=(None, ), dtype='int32', name='question')
# embedded_question = layers.Embedding(32, question_vocab_size)(question_input)
embedded_question = layers.Embedding(question_vocab_size, 32)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)
concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)
# print(concatenated.shape) # 32 + 16 which gives us (?, 48) ... good
answer = layers.Dense(answer_vocab_size, activation='softmax')(concatenated)
model = Model([text_input, question_input], answer)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['acc'])
Y_train,
validation_data=(X_test, Y_test),
callbacks=[es],
epochs=50)
# # LSTM
# In[95]:
X_train, X_test = X_train.reshape(n_len, features,
1), X_test.reshape(n_len_test, features, 1)
# In[97]:
model = Sequential(name="LSTM")
model.add(layers.LSTM(128, input_shape=(X_train.shape[1], 1)))
model.add(layers.Dense(6, activation="softmax"))
# In[98]:
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
# In[99]:
model.fit(X_train,
Y_train,
validation_data=(X_test, Y_test),
callbacks=[es],
epochs=50,
构建一个根据文档内容，标签和标题，预测文档优先级和执行部门的网络
超参
'''
num_words = 2000
num_tags = 12
num_departments = 4
'''输入'''
body_input = tf.keras.layers.Input(shape=(None, ), name='body')
title_input = tf.keras.layers.Input(shape=(None, ), name='title')
tag_input = tf.keras.layers.Input(shape=(num_tags, ), name='tag')
'''嵌入层'''
body_feat = layers.Embedding(num_words, 64)(body_input)
title_feat = layers.Embedding(num_words, 64)(title_input)
'''特征提取层'''
body_feat = layers.LSTM(32)(body_feat)
title_feat = layers.LSTM(32)(title_feat)
features = layers.concatenate([title_feat, body_feat, tag_input])
print(tag_input, body_feat, title_feat, features)
''' 分类层'''
priority_pred = layers.Dense(1, activation='sigmoid',
name='priority')(features)
department_pred = layers.Dense(num_departments,
activation='softmax',
name='department')(features)
print(priority_pred, department_pred)
'''构建模型'''
model = tf.keras.Model(inputs=[body_input, title_input, tag_input],
outputs=[priority_pred, department_pred])
model.summary()
Y = complain_data['Satisfaction'].values
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=0.33, random_state=42)
# print(X_train.shape,Y_train.shape)
# print(X_test.shape,Y_test.shape)
embedding_vector_length = 32
model = Sequential()
model.add(layers.Embedding(max_features,
embedding_vector_length, input_length=maxlen))
model.add(layers.Conv1D(filters=32, kernel_size=3,
padding='same', activation='relu'))
model.add(layers.MaxPooling1D(pool_size=2))
model.add(layers.LSTM(100, recurrent_activation='sigmoid'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
model.summary()
filename = os.path.join(current_dir, 'data', 'complain_model.h5')
is_training = False
if is_training:
model.fit(X_train, Y_train, validation_data=(
X_test, Y_test), epochs=20, batch_size=64)
# Evaluate the model
scores = model.evaluate(X_test, Y_test, verbose=0)
print("Evaluation Accuracy: %.2f%%" % (scores[1]*100))
model.save(filename, save_format='tf')
# if using GPU
if tf.test.is_gpu_available():
print('using GPU')
layer_x = layers.CuDNNLSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer),
kernel_initializer="zeros",
recurrent_initializer="zeros",
bias_initializer="zeros")(layer_x)
# if using CPU
else:
print('using CPU')
layer_x = layers.LSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer),
kernel_initializer="zeros",
recurrent_initializer="zeros",
bias_initializer="zeros")(layer_x)
# # The last output layer uses a softmax activation function
output = layers.Dense(3,
activation='softmax',
kernel_initializer="zeros",
bias_initializer="zeros")(layer_x)
if feature_num == 4:
model = Model(input_tensor, output)
elif feature_num == 5:
model = Model([input_tensor, input_tensor2], output)
Y_train,
validation_data=(X_test, Y_test),
callbacks=[es],
epochs=50)
# # LSTM
# In[ ]:
X_train, X_test = X_train.reshape(n_len, features,
1), X_test.reshape(n_len_test, features, 1)
# In[ ]:
model = Sequential(name="LSTM")
model.add(layers.LSTM(128, input_shape=(X_train.shape[1], 1)))
model.add(layers.Dense(6, activation="softmax"))
# In[ ]:
model.summary()
# In[ ]:
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
# In[ ]:
model.fit(X_train,
X_train /= std_train
X_test -= mean_train
X_test /= std_train
# reshape to be suitable for Keras
y_train = utils.to_categorical(y_train)
y_test = utils.to_categorical(y_test)
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
model = Sequential()
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
input_shape=(None, num_channels * 30 * FreqSample / step),
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
def testSupports_KerasRNNLayers(self): self.assertTrue(self.quantize_registry.supports(l.LSTM(10))) self.assertTrue(self.quantize_registry.supports(l.GRU(10)))
val_loss = history.history['val_loss']
train_acc = history.history['acc']
val_acc = history.history['val_acc']
print(train_loss)
print(val_loss)
print(train_acc)
print(val_acc)
model = Sequential(name='rnn')
model.add(
layers.BatchNormalization(input_shape=(None, num_channels * 30 *
FreqSample / step)))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
def main(thread_num):
# causing allocating 2 gpu devices
# if tf.test.is_gpu_available():
# select gpu device 0,1
if device_type == 'gpu':
os.environ["CUDA_VISIBLE_DEVICES"] = thread_num
task = pd.read_csv("task.csv")
#print(task)
if os.path.isfile("output.csv"):
output_csv = pd.read_csv("output.csv")
else:
output_csv = task
output_csv = output_csv.drop(columns=['data_set'])
#output_csv['train_acc'] = 0.0
output_csv['final_train_loss'] = 100.0
#output_csv['valid_acc'] = 0.0
output_csv['final_valid_loss'] = 100.0
output_csv['best_trade_acc'] = 0.0
output_csv['best_trade_acc_epoch'] = 0
output_csv['best_trade_f1'] = 0.0
output_csv['best_trade_f1_epoch'] = 0
output_csv['best_trade_precision'] = 0.0
output_csv['best_trade_precision_epoch'] = 0
output_csv['best_trade_recall'] = 0.0
output_csv['best_trade_recall_epoch'] = 0
# output_csv['best_trade_loss'] = 100.0
# output_csv['best_trade_loss_epoch'] = 0
output_csv['completed'] = 0
for index, row in task.iterrows():
#if tf.test.is_gpu_available():
if device_type == 'gpu':
if index % 2 != int(thread_num):
continue
completed = output_csv['completed'][index]
if completed == 1:
continue
data_set = int(task['data_set'][index])
load_dir = os.path.join(os.getcwd(), 'data_set/' + str(data_set))
if not os.path.isdir(load_dir):
continue
task_id = int(task['task_id'][index])
input_size = int(task['input'][index])
pred_k = int(task['k'][index])
feature_num = int(task['feature_num'][index])
label_threshold = float(task['label_threshold'][index])
lstm_units = int(task['lstm_units'][index])
lr = float(task['learning_rate'][index])
epsilon = float(task['epsilon'][index])
regularizer = float(task['regularizer'][index])
train_x = np.load(os.path.join(load_dir, 'train_x.npy'))
train_y = np.load(os.path.join(load_dir, 'train_y_onehot.npy'))
valid_x = np.load(os.path.join(load_dir, 'valid_x.npy'))
valid_y = np.load(os.path.join(load_dir, 'valid_y_onehot.npy'))
trade_y = np.load(os.path.join(load_dir, 'trading_valid_y_onehot.npy'))
print('Running experiment {}'.format(task_id))
#clear previous models
clear_session()
model_dir = os.path.join(os.getcwd(), 'load_model')
if os.path.isdir(model_dir):
model_dir = os.path.join(
model_dir,
str(task_id) + '/model/model_epoch_500.h5')
if not os.path.isdir(model_dir):
continue
model = load_model(model_dir)
else:
#input_tensor = Input(shape=(30,4,1))
input_tensor = Input(shape=(input_size, 4, 1))
layer_x = layers.Conv2D(16, (1, 4),
kernel_regularizer=regularizers.l1(
l=regularizer))(input_tensor)
layer_x = layers.BatchNormalization()(layer_x)
layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)
layer_x = layers.Conv2D(
16, (4, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
layer_x = layers.BatchNormalization()(layer_x)
layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)
layer_x = layers.Conv2D(
16, (4, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
layer_x = layers.BatchNormalization()(layer_x)
layer_x = layers.LeakyReLU(alpha=0.01)(layer_x)
#dual input for ohlc+volume
if feature_num == 5:
train_x_ohlc = train_x[:, :, :4, :]
train_x_volume = train_x[:, :, -1:, :]
train_x = [train_x_ohlc, train_x_volume]
valid_x_ohlc = valid_x[:, :, :4, :]
valid_x_volume = valid_x[:, :, -1:, :]
valid_x = [valid_x_ohlc, valid_x_volume]
input_tensor2 = Input(shape=(input_size, 1, 1))
layer_x2 = layers.Conv2D(16, (1, 1),
kernel_regularizer=regularizers.l1(
l=regularizer))(input_tensor2)
layer_x2 = layers.BatchNormalization()(layer_x2)
layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)
layer_x2 = layers.Conv2D(16, (4, 1),
padding='same',
kernel_regularizer=regularizers.l1(
l=regularizer))(layer_x2)
layer_x2 = layers.BatchNormalization()(layer_x2)
layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)
layer_x2 = layers.Conv2D(16, (4, 1),
padding='same',
kernel_regularizer=regularizers.l1(
l=regularizer))(layer_x2)
layer_x2 = layers.BatchNormalization()(layer_x2)
layer_x2 = layers.LeakyReLU(alpha=0.01)(layer_x2)
layer_x = layers.concatenate([layer_x, layer_x2], axis=-1)
# Inception Module
tower_1 = layers.Conv2D(
32, (1, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
tower_1 = layers.BatchNormalization()(tower_1)
tower_1 = layers.LeakyReLU(alpha=0.01)(tower_1)
tower_1 = layers.Conv2D(
32, (3, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(tower_1)
tower_1 = layers.BatchNormalization()(tower_1)
tower_1 = layers.LeakyReLU(alpha=0.01)(tower_1)
tower_2 = layers.Conv2D(
32, (1, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
tower_2 = layers.BatchNormalization()(tower_2)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
tower_2 = layers.Conv2D(
32, (5, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(tower_2)
tower_2 = layers.BatchNormalization()(tower_2)
tower_2 = layers.LeakyReLU(alpha=0.01)(tower_2)
tower_3 = layers.MaxPooling2D((3, 1),
padding='same',
strides=(1, 1))(layer_x)
tower_3 = layers.Conv2D(
32, (1, 1),
padding='same',
kernel_regularizer=regularizers.l1(l=regularizer))(tower_3)
tower_3 = layers.BatchNormalization()(tower_3)
tower_3 = layers.LeakyReLU(alpha=0.01)(tower_3)
layer_x = layers.concatenate([tower_1, tower_2, tower_3], axis=-1)
# concatenate features of tower_1, tower_2, tower_3
layer_x = layers.Reshape((input_size, 96))(layer_x)
#layer_x = layers.Reshape((input_size,feature_num))(input_tensor)
# # 64 LSTM units
#layer_x = layers.LSTM(64)(layer_x)
# if using GPU
if device_type == 'gpu':
print('using GPU')
layer_x = layers.CuDNNLSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
# if using CPU
elif device_type == 'cpu':
print('using CPU')
layer_x = layers.LSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
else:
sys.exit("wrong device type")
# # The last output layer uses a softmax activation function
output = layers.Dense(3, activation='softmax')(layer_x)
if feature_num == 4:
model = Model(input_tensor, output)
elif feature_num == 5:
model = Model([input_tensor, input_tensor2], output)
opt = Adam(lr=lr, epsilon=epsilon)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#model.summary()
save_dir = os.path.join(os.getcwd(), 'result/' + str(task_id))
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
final_train_loss, \
final_valid_loss, \
best_trade_acc, \
best_trade_acc_epoch, \
best_trade_f1, \
best_trade_f1_epoch, \
best_trade_precision, \
best_trade_precision_epoch, \
best_trade_recall, \
best_trade_recall_epoch \
= train_model(model, \
save_dir, \
task_id, \
train_x, \
train_y, \
valid_x, \
valid_y, \
trade_y, \
batch_size=512, \
epochs=3)
with open(os.path.join(save_dir, 'readme.txt'), 'w') as f:
f.write("""'task id = {}\n
input size = {}\n
prediction k = {}\n
feature = {}\n
label threshold = {}\n
lstm units = {}\n
learning rate = {}\n
epsilon = {}\n
regularizer = {}\n
data set = {}'""".format(task_id, \
input_size, \
pred_k, \
feature_num, \
label_threshold, \
lstm_units, \
lr, \
epsilon, \
regularizer, \
data_set))
#output_csv['train_acc'][index] = train_acc
output_csv['final_train_loss'][index] = final_train_loss
#output_csv['valid_acc'][index] = valid_acc
output_csv['final_valid_loss'][index] = final_valid_loss
output_csv['best_trade_acc'][index] = best_trade_acc
output_csv['best_trade_acc_epoch'][index] = best_trade_acc_epoch
output_csv['best_trade_f1'][index] = best_trade_f1
output_csv['best_trade_f1_epoch'][index] = best_trade_f1_epoch
output_csv['best_trade_precision'][index] = best_trade_precision
output_csv['best_trade_precision_epoch'][
index] = best_trade_precision_epoch
output_csv['best_trade_recall'][index] = best_trade_recall
output_csv['best_trade_recall_epoch'][index] = best_trade_recall_epoch
#output_csv['best_trade_loss'] = best_trade_loss
#output_csv['best_trade_loss_epoch'] = best_trade_loss_epoch
output_csv['completed'][index] = 1
output_csv.to_csv('output.csv')
#layer_x = layers.Reshape((input_size,feature_num))(input_tensor)
# # 64 LSTM units
#layer_x = layers.LSTM(64)(layer_x)
# if using GPU
if tf.test.is_gpu_available():
print('using GPU')
layer_x = layers.CuDNNLSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
# if using CPU
else:
print('using CPU')
layer_x = layers.LSTM(
lstm_units,
kernel_regularizer=regularizers.l1(l=regularizer))(layer_x)
# # The last output layer uses a softmax activation function
output = layers.Dense(3, activation='softmax')(layer_x)
if feature_num == 4:
model = Model(input_tensor, output)
elif feature_num == 5:
model = Model([input_tensor, input_tensor2], output)
opt = Adam(lr=lr, epsilon=1
) # learning rate and epsilon are the same as paper DeepLOB
model.compile(loss='categorical_crossentropy',
optimizer=opt,
xtrain = tokenizer.texts_to_sequences(x_train)
xtest = tokenizer.texts_to_sequences(x_test)
maxlen = 10
xtrain = pad_sequences(xtrain, padding='post', maxlen=maxlen)
xtest = pad_sequences(xtest, padding='post', maxlen=maxlen)
vocab_size = len(tokenizer.word_index) + 1
embedding_dim = 50
model = Sequential()
model.add(
layers.Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
input_length=maxlen))
model.add(layers.LSTM(units=50, return_sequences=True))
model.add(layers.LSTM(units=10))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(8))
model.add(layers.Dense(1, activation="sigmoid"))
model.compile(optimizer="adam",
loss="binary_crossentropy",
metrics=['accuracy'])
model.summary()
model.fit(xtrain, y_train, epochs=20, batch_size=16, verbose=True)
model.save('my_model')
print('model saved!')
loss, acc = model.evaluate(xtrain, y_train, verbose=True)
print("Training Accuracy: ", acc)
from tensorflow.python.keras import Input
from tensorflow.python.keras import Model
from tensorflow.python.keras import layers
import tensorflow as tf
main_input = Input(shape=(100, ), dtype='int32', name='main_input')
x = layers.Embedding(output_dim=512, input_dim=10000,
input_length=100)(main_input)
lstm_out = layers.LSTM(32)(x)
auxiliary_output = layers.Dense(1, activation='sigmoid',
name='aux_output')(lstm_out)
aux_input = Input(shape=(5, ), name='aux_input')
x = layers.concatenate([lstm_out, aux_input])
x = layers.Dense(64, activation='relu')(x)
x = layers.Dense(64, activation='relu')(x)
x = layers.Dense(64, activation='relu')(x)
main_output = layers.Dense(1, activation='sigmoid', name='main_output')(x)
model = Model(inputs=[main_input, aux_input],
outputs=[main_output, auxiliary_output])
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
loss_weights=[1., 0.2])