def _create_decoder(encoder_outputs, encoder_state_h, encoder_state_c, step):
decoder_inputs = tf.repeat(
input=tf.expand_dims(encoder_state_h, axis=1),
repeats=step.hyperparams['output_length'],
axis=1
)
decoder = LSTM(
units=step.hyperparams['hidden_dim'],
activation='relu',
return_state=False,
return_sequences=True
)
decoder_outputs = decoder(decoder_inputs, initial_state=[encoder_state_h, encoder_state_c])
decoder_outputs = _create_attention_mechanism(
decoder_outputs=decoder_outputs,
encoder_outputs=encoder_outputs,
step=step
)
# (batch_size, output_length, output_dim)
return decoder_outputs
def build_model(input_dim, max_length, vocab_size, embedding_dim,
embedding_matrix):
inputs1 = Input(shape=(input_dim, ))
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation='relu')(fe1)
inputs2 = Input(shape=(max_length, ))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=False)(inputs2)
se2 = Dropout(0.5)(se1)
se3 = LSTM(256)(se2)
# decoder1 = Concatenate()([fe2, se3])
decoder1 = add([fe2, se3])
decoder2 = Dense(256, activation='relu')(decoder1)
# attention_probs = Dense(256, activation='softmax', name='attention_probs')(decoder2)
# attention_mul = merge.Multiply()([decoder1, attention_probs])
outputs = Dense(vocab_size, activation='softmax')(decoder2)
model = Model(inputs=[inputs1, inputs2], outputs=outputs)
model.layers[2].set_weights([embedding_matrix])
model.layers[2].trainable = False
model.compile(loss='categorical_crossentropy', optimizer='adam')
return model
def create_model():
model = Sequential()
model.add(
LSTM(16, input_shape=(2, 2), return_sequences=True, activation='tanh'))
model.add(LSTM(32, return_sequences=True, activation='tanh'))
model.add(LSTM(64, return_sequences=True, activation='tanh'))
model.add(LSTM(128, return_sequences=True, activation='tanh'))
model.add(LSTM(64, return_sequences=True, activation='tanh'))
model.add(Dropout(.15))
model.add(LSTM(32, return_sequences=True, activation='tanh'))
model.add(Dense(16, activation='sigmoid'))
model.add(LSTM(8, return_sequences=True, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.summary()
rms = RMSprop(lr=.01)
model.compile(optimizer=rms,
loss='categorical_crossentropy',
metrics=['accuracy'])
if "weights.h5" in os.listdir(".") and train_mode == False:
model.load_weights(str("weights.h5"))
print("loading saved network: weights.h5")
return model
def buildModel(classes, features, cellNeurons, cellDropout, denseDropout, denseNeurons, sequenceLength, stacked=False, bidirectional=False, l2=0.0):
model = Sequential()
model.add(TimeDistributed(Dense(features,
kernel_initializer='identity',
bias_initializer='zeros',
name='customNn',
activation=None), input_shape=(sequenceLength, features), name='td', trainable=False))
if bidirectional:
if stacked:
model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, return_sequences=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat'))
model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn_2nd_layer', trainable=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat'))
else:
model.add(Bidirectional(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, kernel_regularizer=regularizers.l2(l2)), merge_mode='concat'))
else:
if stacked:
model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, return_sequences=True, kernel_regularizer=regularizers.l2(l2)))
model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn_2nd_layer', trainable=True, kernel_regularizer=regularizers.l2(l2)))
#model.add(Attention(name='attention', trainable=True))
else:
model.add(LSTM(cellNeurons, recurrent_dropout=cellDropout, name='rnn', trainable=True, kernel_regularizer=regularizers.l2(l2)))
model.add(Dense(denseNeurons, name='nn', trainable=True, kernel_regularizer=regularizers.l2(l2)))
model.add(Dropout(denseDropout, name='nn_dropout', trainable=True))
model.add(Dense(classes, activation="softmax", name='output_softmax', trainable=True, kernel_regularizer=regularizers.l2(l2)))
#model.summary()
if onTpu:
model.compile(loss="categorical_crossentropy",
optimizer=tf.train.AdamOptimizer(learning_rate=0.001),
metrics=["accuracy"])
multi_model = toTpuModel(model)
else:
multi_model = toMultiGpuModel(model)
multi_model.compile(loss="categorical_crossentropy",
optimizer=optimizers.Adam(lr=0.001),
metrics=["accuracy"])
return model, multi_model
#每组采样300条数据用于训练 index0 = np.random.permutation(np.where(y == 0)[0]) index1 = np.random.permutation(np.where(y == 1)[0]) #准备训练数据 x_train = np.concatenate((x[index0[0:300]], x[index1[0:300]]), axis=0) y_train = np.concatenate((y[index0[0:300]], y[index1[0:300]]), axis=0) #准备测试数据 x_test = np.concatenate((x[index0[300:400]], x[index1[300:400]]), axis=0) y_test = np.concatenate((y[index0[300:400]], y[index1[300:400]]), axis=0) #定义神经网络 model = tf.keras.Sequential() model.add(LSTM(50, input_shape=(x_train.shape[1], x_train.shape[2]))) model.add(Dense(1024, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(128, activation='relu')) model.add(Dropout(0.25)) model.add(Dense(1, activation='sigmoid')) # 输出层 model.summary() # 定义优化器 sgd = tf.keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) # 整合模型 model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])
# reshape into X=t and Y=t+1
lags = 3
X_train, y_train = prepare_data(train, lags)
X_test, y_test = prepare_data(test, lags)
# reshape input to be [samples, time steps, features]
X_train = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1]))
X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))
tbCallBack = tf.keras.callbacks.TensorBoard(log_dir='./tmp/model_graph',
write_graph=True)
# create and fit the LSTM network
mdl = Sequential()
mdl.add(Dense(4, input_shape=(1, lags), activation='relu'))
mdl.add(LSTM(6, activation='relu'))
mdl.add(Dense(1, activation='relu'))
mdl.compile(loss='mean_squared_error', optimizer='adam')
mdl.fit(X_train,
y_train,
epochs=10000,
batch_size=10,
verbose=2,
callbacks=[tbCallBack])
# make predictions
train_predict = mdl.predict(X_train)
test_predict = mdl.predict(X_test)
# invert transformation
train_predict = scaler.inverse_transform(train_predict)
def keras_BiDirect_estimator(model_dir,
config,
learning_rate,
dropout_rate,
embedding_dim,
embedding_path=None,
word_index=None):
model = models.Sequential()
num_features = min(len(word_index) + 1, TOP_K)
# If pre-trained embedding is used add weights to the embeddings layer
# and set trainable to input is_embedding_trainable flag.
if embedding_path != None:
embedding_matrix = get_embedding_matrix(word_index, embedding_path,
embedding_dim)
is_embedding_trainable = True # set to False to freeze embedding weights
model.add(
Embedding(input_dim=num_features,
output_dim=embedding_dim,
input_length=MAX_SEQUENCE_LENGTH,
weights=[embedding_matrix],
trainable=is_embedding_trainable))
else:
model.add(
Embedding(input_dim=num_features,
output_dim=embedding_dim,
input_length=MAX_SEQUENCE_LENGTH))
model.add(SpatialDropout1D(dropout_rate))
model.add(
Bidirectional(
LSTM(embedding_dim * 2,
dropout=dropout_rate,
recurrent_dropout=0.2,
return_sequences=True)))
model.add(
Bidirectional(
LSTM(int(embedding_dim / 2),
dropout=dropout_rate,
recurrent_dropout=0.2)))
model.add(Dropout(rate=dropout_rate))
model.add(
Dense(len(CLASSES),
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activation='softmax'))
# Compile model with learning parameters.
optimizer = tf.keras.optimizers.Adam(lr=learning_rate)
model.compile(
optimizer=optimizer,
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
estimator = tf.keras.estimator.model_to_estimator(keras_model=model,
model_dir=model_dir,
config=config)
return estimator
def __init__(self, fl, mode, hparams):
"""
Initialises new DNN model based on input features_dim, labels_dim, hparams
:param features_dim: Number of input feature nodes. Integer
:param labels_dim: Number of output label nodes. Integer
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function.
hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes.
"""
# self.features_dim = fl.features_c_dim
# self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.features_dim = fl.features_c_dim + 1 # 1 for the positional argument
self.labels_dim = 1
self.numel = fl.labels.shape[1] + 1
self.hparams = hparams
self.mode = mode
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Selection of model
if mode == 'ann':
model = ann(self.features_dim, self.labels_dim, self.hparams)
x = model(features_in)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'ann2':
model_1 = ann(self.features_dim, 50, self.hparams)
x = model_1(features_in)
model_end = ann(50, 50, self.hparams)
end = model_end(x)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
model_2 = ann(50, self.labels_dim - 1, self.hparams)
x = model_2(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'ann3':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(0))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv1':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared' + str(1))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
#x = BatchNormalization()(x)
x = Dense(units=19,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
#x = BatchNormalization()(x)
x = Reshape(target_shape=(19, 1))(x)
x = Conv1D(filters=hparams['filters'],
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = BatchNormalization()(x)
x = Conv1D(filters=hparams['filters'] * 2,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = Conv1D(filters=hparams['filters'] * 4,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(19, ))(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv2':
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=80,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Reshape(target_shape=(80, 1))(x)
x = Conv1D(filters=8,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Conv1D(filters=16,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(20, ))(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'lstm':
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
x = RepeatVector(n=20)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = TimeDistributed(Dense(1))(x)
x = Reshape(target_shape=(20, ))(x)
'''
x = Permute((2,1))(x)
x = GlobalAveragePooling1D()(x)
'''
self.model = Model(inputs=features_in, outputs=[end_node, x])
optimizer = Adam(clipnorm=1)
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
train_ratio = 0.7 train_size = 7000 test_ratio = 0.3 test_size = 3000 x_train, x_test = x_in_rs[:train_size], x_in_rs[(train_size):(sample_size)] y_train, y_test = y_true[:train_size], y_true[(train_size):(sample_size)] data_dim = 4 timesteps = 1 data_out = 2 # Tensorflow model model = Sequential() # Input Layer model.add(LSTM(32, activation = 'tanh', return_sequences=True, input_shape=(timesteps, data_dim))) # model.add(Dense(32, kernel_initializer='normal', activation='relu', input_dim=data_dim)) # Hidden Layer model.add(Dense(64, kernel_initializer='normal', activation='relu')) # Hidden Layer model.add(Dense(128, kernel_initializer='normal', activation='relu')) # Hidden Layer model.add(Dense(256, kernel_initializer='normal', activation='relu')) # The Output Layer : model.add(Dense(data_out, kernel_initializer='normal',activation='linear')) # Compile the network : model.compile(loss='mae', optimizer='adam', metrics=[mae]) model.summary() # reshape x_train, x_test to fit input in LSTM layers
from sklearn.model_selection import train_test_split
# 90%的样本用来训练，剩余10%用来测试
X_train, X_test, y_train, y_test = train_test_split(train_pad, train_target, test_size=0.1, random_state=12)
# 查看训练样本，确认无误
print(reverse_tokens(X_train[35]))
print('class: ', y_train[35])
# 用LSTM对样本进行分类
model = Sequential()
# 模型第一层为embedding
model.add(Embedding(num_words, embedding_dim, weights=[embedding_matrix], input_length=max_tokens, trainable=False))
model.add(Bidirectional(LSTM(units=64, return_sequences=True)))
model.add(LSTM(units=16, return_sequences=False))
model.add(Dense(1, activation='sigmoid'))
# 我们使用adam以0.001的learning rate进行优化
optimizer = Adam(lr=1e-3)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# 我们来看一下模型的结构，一共90k左右可训练的变量
model.summary()
# 建立一个权重的存储点
path_checkpoint = 'sentiment_checkpoint.keras'
checkpoint = ModelCheckpoint(filepath=path_checkpoint, monitor='val_loss', verbose=1, save_weights_only=True, save_best_only=True)
def __init__(self,
fields=None,
variables=None,
hidden_layers=None,
activation="tanh",
output_activation="linear",
rnn_type="SimpleRNN",
recurrent_activation="tanh",
kernel_initializer=None,
bias_initializer=None,
kernel_regularizer=None,
bias_regularizer=None,
dtype=None,
trainable=True,
**kwargs):
# check data-type.
if dtype is None:
dtype = K.floatx()
elif not K.floatx() == dtype:
K.set_floatx(dtype)
# prepare hidden layers.
if hidden_layers is None:
hidden_layers = []
else:
hidden_layers = to_list(hidden_layers)
# check for copy constructor.
if all([x in kwargs for x in ('inputs', 'outputs', 'layers')]):
self._inputs = kwargs['inputs'].copy()
self._outputs = kwargs['outputs'].copy()
self._layers = kwargs['layers'].copy()
self._set_model()
return
# prepare kernel initializers.
activations, def_biasinit, def_kerinit = \
prepare_default_activations_and_initializers(
len(hidden_layers) * [activation] + [output_activation]
)
if kernel_initializer is None:
kernel_initializer = def_kerinit
elif isinstance(kernel_initializer, (float, int)):
kernel_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=kernel_initializer)
else:
kernel_initializer = [
kernel_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare bias initializers.
if bias_initializer is None:
bias_initializer = def_biasinit
elif isinstance(bias_initializer, (float, int)):
bias_initializer = default_weight_initializer(
len(hidden_layers) * [activation] + [output_activation],
'constant',
scale=bias_initializer)
else:
bias_initializer = [
bias_initializer for l in len(hidden_layers) * [activation] +
[output_activation]
]
# prepare regularizers.
kernel_regularizer = default_regularizer(kernel_regularizer)
bias_regularizer = default_regularizer(bias_regularizer)
# prepares fields.
fields = to_list(fields)
if all([isinstance(fld, str) for fld in fields]):
output_fields = [
RNNField(
name=fld,
dtype=dtype,
kernel_initializer=kernel_initializer[-1],
bias_initializer=bias_initializer[-1],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
) for fld in fields
]
elif all([validations.is_field(fld) for fld in fields]):
output_fields = fields
else:
raise TypeError('Please provide a "list" of field names of' +
' type "String" or "Field" objects.')
# prepare inputs/outputs/layers.
inputs = []
layers = []
variables = to_list(variables)
if all([isinstance(var, RNNFunctional) for var in variables]):
for var in variables:
inputs += var.outputs
# for var in variables:
# for lay in var.layers:
# layers.append(lay)
else:
raise TypeError(
"Input error: Please provide a `list` of `Functional`s. \n"
"Provided - {}".format(variables))
# prepare hidden layers.
if hidden_layers is None:
hidden_layers = []
else:
hidden_layers = to_list(hidden_layers)
# Check and convert activation functions to proper format.
assert not isinstance(activation, list), \
'Expected an activation function name not a "list". '
afunc = get_activation(activation)
# Input layers.
if len(inputs) == 1:
net_input = inputs[0]
else:
layer = Concatenate(name=graph_unique_name('conct'))
net_input = layer(inputs)
# Define the networks.
net = [net_input]
assert len(
hidden_layers) > 0, 'Minimum of 1 RNN hidden layer is needed.'
# Adding hidden layers
for nLay, nNeuron in enumerate(hidden_layers):
if nLay < 1000:
# First layer starts with RNN.
if rnn_type == 'LSTM':
layer = LSTM(nNeuron,
return_sequences=True,
recurrent_activation=recurrent_activation,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=graph_unique_name(
"LSTM{:d}b_".format(nNeuron)))
elif rnn_type == 'SimpleRNN':
layer = SimpleRNN(
nNeuron,
return_sequences=True,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=graph_unique_name("SRNN{:d}b_".format(nNeuron)))
else:
raise ValueError('Invalid entry for `rnn_type` -- '
'accepts from (`SimpleRNN`, `LSTM`).')
else:
# Add the dense layer.
layer = Dense(nNeuron,
kernel_initializer=kernel_initializer[nLay],
bias_initializer=bias_initializer[nLay],
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
name=graph_unique_name("D{:d}b".format(nNeuron)))
layers.append(layer)
net[-1] = layer(net[-1])
# Apply the activation.
if afunc.__name__ != 'linear':
layer = activations[nLay]
layers.append(layer)
net[-1] = layer(net[-1])
# store output layers.
for out in output_fields:
layers.append(out)
# Assign to the output variable
if len(net) == 1:
net_output = net[0]
else:
raise ValueError("Legacy for Enrichment: Must be updated. ")
layer = Concatenate(name=graph_unique_name("{}_".format("conct")))
net_output = layer(net)
# check output activation functions.
output_func = get_activation(output_activation)
# Define the final outputs of each network
outputs = []
for out in output_fields:
# add the activation on the output.
if output_func.__name__ != 'linear':
layer = activations[-1]
layers.append(layer)
outputs.append(layer(out(net_output)))
else:
outputs.append(out(net_output))
self._inputs = inputs
self._outputs = outputs
self._layers = layers
self._set_model()
scaler = MinMaxScaler(feature_range=(0, 1))
dataset = scaler.fit_transform(dataset)
# split into train and test sets
train_size = int(len(dataset) * 0.67)
test_size = len(dataset) - train_size
train, test = dataset[0:train_size, :], dataset[train_size:len(dataset), :]
# reshape into X=t and Y=t+1
look_back = 1
trainX, trainY = create_dataset(train, look_back)
testX, testY = create_dataset(test, look_back)
# reshape input to be [samples, time steps, features]
trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
# create and fit the LSTM network
model = Sequential()
model.add(LSTM(4, input_shape=(1, look_back)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(trainX, trainY, epochs=100, batch_size=1, verbose=2)
# make predictions
trainPredict = model.predict(trainX)
testPredict = model.predict(testX)
# invert predictions
trainPredict = scaler.inverse_transform(trainPredict)
trainY = scaler.inverse_transform([trainY])
testPredict = scaler.inverse_transform(testPredict)
testY = scaler.inverse_transform([testY])
# calculate root mean squared error
trainScore = math.sqrt(mean_squared_error(trainY[0], trainPredict[:, 0]))
print('Train Score: %.2f RMSE' % (trainScore))
testScore = math.sqrt(mean_squared_error(testY[0], testPredict[:, 0]))
#two categorical arrays, one for each side of the functional network
#testY1, testY2 = np.hsplit(testY,2)
#testY1 = to_categorical(testY1)
#testY2 = to_categorical(testY2)
valY1, valY2 = np.hsplit(valY,2)
valY1 = to_categorical(valY1)
valY2 = to_categorical(valY2)
trainY1, trainY2 = np.hsplit(trainY,2)
trainY1 = to_categorical(trainY1)
trainY2 = to_categorical(trainY2)
#build functional model
visible =Input(shape=(trainX.shape[1],trainX.shape[2]))
hidden1 = LSTM(32, return_sequences=True, name='firstLSTMLayer')(visible)
hidden2 = LSTM(16, name='secondLSTMLayer',return_sequences=True)(hidden1)
#left branch decides second agent action
hiddenLeft = LSTM(10, name='leftBranch')(hidden2)
agent2 = Dense(5,activation='softmax',name='agent2classifier')(hiddenLeft)
#right branch decides third agent action
hiddenRight = LSTM(10, name='rightBranch')(hidden2)
agent3 = Dense(5,activation='softmax',name='agent3classifier')(hiddenRight)
model = Model(inputs=visible,outputs=[agent2,agent3])
model.compile(optimizer='adam',
loss={'agent2classifier': 'categorical_crossentropy',
'agent3classifier': 'categorical_crossentropy'},
metrics={'agent2classifier': ['acc'],
'agent3classifier': ['acc']})
vocab = pd.read_csv(os.path.join(training_dir, "vocab.csv"), header=None, names=None)
# Labels are in the first column
train_y = train_data.iloc[:, 0]
train_x = train_data.iloc[:, 1:]
max_answer_len = train_x.values.shape[1]
# Build Model
model = Sequential()
model.add(Embedding(len(vocab), args.embedding_size, input_length=max_answer_len))
model.add(Dropout(args.dropout))
if args.flatten:
model.add(Flatten())
model.add(Reshape((1, args.embedding_size * max_answer_len)))
if args.lstm_dim_2:
model.add(LSTM(args.lstm_dim_1, return_sequences=True))
model.add(LSTM(args.lstm_dim_2, return_sequences=False))
else:
model.add(LSTM(args.lstm_dim_1, return_sequences=False))
model.add(Dropout(args.dropout))
model.add(Dense(1, activation="linear"))
optimizer = AdamOptimizer()
model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['acc'])
# Train the model
model.fit(train_x, train_y, epochs=args.epochs, verbose=0)
# Validate
test_y = test_data.iloc[:, 0]
test_x = test_data.iloc[:, 1:]
# # Keras를 활용한 CNN-LSTM 모델 생성 및 학습
# In[61]:
model1=Sequential() #input_shape=(36=오전 9시부터 5시까지의 15분 단위 데이터 갯수 ,7 = 일사량 예측을 위한 변수 갯수)
model1.add(TimeDistributed(Conv1D(32, kernel_size=2, activation='relu'), input_shape=(xB.shape[1],xB.shape[2],1)))
model1.add(BatchNormalization()) #배치 정규화를 통한 과적합 방지
model1.add(Dropout(0.3)) #과적합을 막기위한 dropout
model1.add(TimeDistributed(MaxPooling1D(pool_size=2, strides=2)))
#컨볼루션 연산을 통해 생성된 데이터의 지역적 특징 중 가장 큰 값을 반영하여 특징을 함축하는 MaxPooling층을 사용하여 지역적 특징을 연산합니다.
#strides가 2이므로 입력 변수 갯수가 반으로 줄어듭니다.
model1.add(Reshape((36,96), input_shape=(36,3,32))) #LSTM에 넣기 위해 3차원 데이터를 2차원으로 변환.
model1.add(LSTM(32, input_shape=(36,96),return_sequences=True)) #stacked LSTM(2층 짜리) 구현
model1.add(Activation('relu')) #활성화 함수로 ReLu적용
model1.add(Dropout(0.3))#과적합을 막기위한 dropout
model1.add(LSTM(32)) #두 번째 LSTM
model1.add(Activation('relu'))#활성화 함수로 ReLu적용
model1.add(Dropout(0.3))#과적합을 막기위한 dropout
model1.add(Dense(36))# 하루 단위 결과 출력
model1.compile(loss='mse', optimizer='adam') #손실함수 MSE, 최적화 함수 ADAM 적용
model1.summary()
model1.fit(xB, yB, epochs=370, verbose=1)
#기존의 기상청 데이터와 일사량데이터를 훈련데이터로 하여 학습 반복 횟수 370으로 설정하여 모델 학습 진행.
# In[62]:
def model_build(FEATURE_NUM): # 模型构建层
'''
model = Sequential() # 序贯模型（Sequential）单输入单输出，一条路通到底，层与层之间只有相邻关系，没有跨层连接
# model.add(LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))) # 单向，在此改双向
model.add(Bidirectional(GRU(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')) # 双向GRU 第一层要指定数据输入的形状
# model.add(Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)),
# merge_mode='concat')) # 双向LSTM 第一层要指定数据输入的形状，改双层此处多加一句return返回ndim=3的序列
model.add(BatchNormalization()) # 加入正则化
model.add(Dropout(0.2)) # 随机选取，防止过拟合
# model.add(Bidirectional(LSTM(30))) # 第二层双向LSTM
model.add(Bidirectional(GRU(30))) # 第二层双向GRU
model.add(BatchNormalization()) # 加入正则化
model.add(Dropout(0.2)) # 随机选取，防止过拟合
model.add(Dense(FEATURE_NUM, activation='softmax')) # 是Keras定义网络层的基本方法，有FEATURE_NUM个节点，激活函数是softmax
model.add(Flatten()) # 比单向多出来一句
# Flatten层用来将输入“压平”，即把多维的输入一维化，常用在从卷积层到全连接层的过渡。Flatten不影响batch的大小
model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])
model.build((None, 90, 3)) # 告诉模型输入的格式 其中90是lstm的time_step ，3是input_dim，none这里个人认为代表样本数
model.summary()
'''
# 完整版 CNN-Att-BiLSTM
'''
input = Input(shape=(TIMESERIES_LENGTH, 3)) # 1. 输入shape
conv1 = Conv1D(filters=16, kernel_size=10, activation='relu')(input) # 2. for input1 主要用的卷积层
LSTM1 = Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(conv1) # 3. 第一层双向GRU
NormalOut = BatchNormalization()(LSTM1) # 4. 正则化，防止过拟合
DropOut = Dropout(0.2)(NormalOut) # 5. dropout，防止过拟合
attention_mul = attention_3d_block(DropOut) # 6. 注意力机制
BiLSTM2 = Bidirectional(LSTM(30))(attention_mul) # 7. 第二层双向GRU
NormalOut = BatchNormalization()(BiLSTM2) # 8. 正则化，防止过拟合
DropOut = Dropout(0.2)(NormalOut) # 9. dropout，防止过拟合
'''
input = Input(shape=(TIMESERIES_LENGTH, 3)) # 1. 输入shape
# conv1 = Conv1D(filters=48, kernel_size=6, strides=1, activation='relu')(input) # for input1
# conv1 = Conv1D(filters=16, kernel_size=10, activation='relu')(input) # 2. for input1 主要用的卷积层
# lstm1 = LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3))(conv1) # 单层LSTM
# attention_mul = attention_3d_block(conv1)
BiLSTM1 = Bidirectional(LSTM(30,
return_sequences=True,
input_shape=(TIMESERIES_LENGTH, 3)),
merge_mode='concat')(input) # 3. 第一层双向LSTM
# BiLSTM1 = Bidirectional(LSTM(30, return_sequences=True, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(
# conv1) # 第一层双向LSTM
# attention_mul = attention_3d_block(BiLSTM1) # 6. 注意力机制
# LSTM1 = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(attention_mul) # 单层LSTM
# BiLSTM2 = Bidirectional(LSTM(30, input_shape=(TIMESERIES_LENGTH, 3)), merge_mode='concat')(attention_mul) # 单层双向GRU
# BiGRU1 = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(input) # 单层LSTM
# conv2 = Conv1D(filters=16, kernel_size=10, activation='relu')(attention_mul)
NormalOut = BatchNormalization()(BiLSTM1) # 4. 正则化，防止过拟合
DropOut = Dropout(0.2)(NormalOut) # 5. dropout，防止过拟合
attention_mul = attention_3d_block(DropOut) # 6. 注意力机制
# # BiGRU2 = Bidirectional(GRU(30))(attention_mul) # 7. 第二层双向GRU
BiLSTM12 = Bidirectional(LSTM(30))(attention_mul) # 第二层双向LSTM
# # LSTMOut = LSTM(30, input_shape=(TIMESERIES_LENGTH, 3))(conv1)
# NormalOut = BatchNormalization()(BiLSTM12) # 8. 正则化，防止过拟合
NormalOut = BatchNormalization()(BiLSTM12) # 正则化，防止过拟合
DropOut = Dropout(0.2)(NormalOut) # 9. dropout，防止过拟合
# 分界线
DenseOut = Dense(FEATURE_NUM, activation='softmax')(
DropOut) # 10. 是Keras定义网络层的基本方法，有FEATURE_NUM个节点，激活函数是softmax
model = Model(inputs=input, outputs=DenseOut)
model.compile(optimizer=Adam(lr=0.001),
loss='categorical_crossentropy',
metrics=['accuracy']) # 交叉熵损失函数
model.build((None, 90, 3))
model.summary()
return model
print(X_train.shape)
print(X_test.shape)
"""(15, 40, 20) 15个样本 40个特征 每个特征用20词向量表示"""
#拉直形状 (15, 40, 20)=>(15, 40*20) (6, 40, 20)=>(6, 40*20)
X_train = X_train.reshape(len(y_train), maxlen * max_features)
X_test = X_test.reshape(len(y_test), maxlen * max_features)
#--------------------------------建模与训练-------------------------------
model = Sequential()
#构建Embedding层 128代表Embedding层的向量维度
model.add(Embedding(max_features, 128))
#构建LSTM层
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
#构建全连接层
#注意上面构建LSTM层时只会得到最后一个节点的输出,如果需要输出每个时间点的结果需将return_sequences=True
model.add(Dense(units=1, activation='sigmoid'))
#模型可视化
model.summary()
#激活神经网络
model.compile(
optimizer='rmsprop', #RMSprop优化器
loss='binary_crossentropy', #二元交叉熵损失
metrics=['acc'] #计算误差或准确率
)
sys.exit(0)
else:
grid_df.loc[idx, 'timestamp'] = curr_time
train_generator = crop_generator(input_path=tr_path, batch_size=batch_size, mode="train", do_shuffle=True, epsilon=0)
val_generator = crop_generator(input_path=val_path, batch_size=batch_size, mode="val", do_shuffle=True, epsilon=0)
# temporal part starts here
temporal_input_layer = Input(batch_shape = (None, 23, 1), name='time_input_layer')
# add an LSTM layer
lstm_1 = LSTM(100, input_shape=(23,1), dropout=dropout_rate)(temporal_input_layer)
# add a dense layer
ts_output = Dense(32, activation="relu")(lstm_1)
# temporal part ends here
# and a softmax layer -- num_classes
predictions = Dense(num_classes, activation='softmax')(ts_output)
optimizer = Adam(lr=learning_rate)
# this is the model we will train
model = Model(inputs= temporal_input_layer, outputs=predictions)
def test_delete_channels_lstm(channel_index):
layer = LSTM(9, return_sequences=True)
recursive_test_helper(layer, channel_index)
EPOCHS = 10
EMBED_DIM = 64
BiRNN_UNITS = 200
vacab_size = vocabulary_lookuper.size()
tag_size = tag_lookuper.size()
# model = Sequential()
# model.add(Embedding(vacab_size, EMBED_DIM, mask_zero=True))
# model.add(Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True)))
# model.add(CRF(tag_size))
raw_input = layers.Input(shape=(MAX_LEN,))
embedding_layer = Embedding(vacab_size, EMBED_DIM, mask_zero=True)(raw_input)
bilstm_layer = Bidirectional(LSTM(BiRNN_UNITS // 2, return_sequences=True))(embedding_layer)
crf_layer = CRF(
units=tag_size,
transition_constraint_matrix=constraint_table
)
dynamic_constraint_input = layers.Input(shape=(intent_number,))
output_layer = crf_layer([bilstm_layer, dynamic_constraint_input])
model = models.Model([raw_input, dynamic_constraint_input], output_layer)
# print model summary
model.summary()
def __init__(self, fl, mode, hparams):
"""
Initialises new DNN model based on input features_dim, labels_dim, hparams
:param features_dim: Number of input feature nodes. Integer
:param labels_dim: Number of output label nodes. Integer
:param hparams: Dict containing hyperparameter information. Dict can be created using create_hparams() function.
hparams includes: hidden_layers: List containing number of nodes in each hidden layer. [10, 20] means 10 then 20 nodes.
"""
self.features_dim = fl.features_c_dim
self.labels_dim = fl.labels_dim # Assuming that each task has only 1 dimensional output
self.hparams = hparams
self.mode = mode
self.normalise_labels = fl.normalise_labels
self.labels_scaler = fl.labels_scaler
features_in = Input(shape=(self.features_dim, ),
name='main_features_c_input')
# Selection of model
if mode == 'ann':
model = ann(self.features_dim, self.labels_dim, self.hparams)
x = model(features_in)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'ann2':
model_1 = ann(self.features_dim, 50, self.hparams)
x = model_1(features_in)
model_end = ann(50, 50, self.hparams)
end = model_end(x)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
model_2 = ann(50, self.labels_dim - 1, self.hparams)
x = model_2(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'ann3':
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(0))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
# x = BatchNormalization()(x)
x = Dense(units=self.labels_dim,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Final')(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv1':
if fl.label_type == 'gf20':
final_dim = 20
else:
final_dim = 19
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='shared' + str(1))(features_in)
x = Dense(units=hparams['pre'],
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
#x = BatchNormalization()(x)
x = Dense(units=final_dim,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_set_19')(x)
#x = BatchNormalization()(x)
x = Reshape(target_shape=(final_dim, 1))(x)
x = Conv1D(filters=hparams['filters'],
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = BatchNormalization()(x)
x = Conv1D(filters=hparams['filters'] * 2,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = Conv1D(filters=hparams['filters'] * 4,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(final_dim, ))(x)
self.model = Model(inputs=features_in, outputs=x)
elif mode == 'conv2':
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=10,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=80,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Reshape(target_shape=(80, 1))(x)
x = Conv1D(filters=8,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
x = Conv1D(filters=16,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = MaxPooling1D(pool_size=2)(x)
#x = Permute((2,1))(x)
#x = GlobalAveragePooling1D()(x)
x = TimeDistributed(Dense(1, activation='linear'))(x)
x = Reshape(target_shape=(20, ))(x)
self.model = Model(inputs=features_in, outputs=[end_node, x])
elif mode == 'lstm':
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(1))(features_in)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Shared_e_' + str(2))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(1))(x)
end = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Dense_e_' + str(2))(end)
end_node = Dense(units=1,
activation='linear',
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='output_layer')(end)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(1))(x)
x = Dense(units=20,
activation=hparams['activation'],
kernel_regularizer=regularizers.l1_l2(
l1=hparams['reg_l1'], l2=hparams['reg_l2']),
name='Pre_' + str(2))(x)
x = RepeatVector(n=20)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = LSTM(units=30, activation='relu', return_sequences=True)(x)
x = TimeDistributed(Dense(1))(x)
x = Reshape(target_shape=(20, ))(x)
'''
x = Permute((2,1))(x)
x = GlobalAveragePooling1D()(x)
'''
self.model = Model(inputs=features_in, outputs=[end_node, x])
optimizer = Adam(learning_rate=hparams['learning_rate'], clipnorm=1)
def weighted_mse(y_true, y_pred):
loss_weights = np.sqrt(np.arange(1, 20))
#loss_weights = np.arange(1, 20)
return K.mean(K.square(y_pred - y_true) * loss_weights, axis=-1)
def haitao_error(y_true, y_pred):
diff = K.abs(
(y_true - y_pred) /
K.reshape(K.clip(K.abs(y_true[:, -1]), K.epsilon(), None),
(-1, 1)))
return 100. * K.mean(diff, axis=-1)
if hparams['loss'] == 'mape':
self.model.compile(optimizer=optimizer,
loss=MeanAbsolutePercentageError())
elif hparams['loss'] == 'haitao':
self.model.compile(optimizer=optimizer, loss=haitao_error)
elif hparams['loss'] == 'mse':
self.model.compile(optimizer=optimizer, loss='mean_squared_error')
# Model variables
gpus = 1
batch_size = 1024 * gpus
n_epoch = 100
n_hidden = 50
# Define the shared model
x = Sequential()
x.add(
Embedding(len(embeddings),
embedding_dim,
weights=[embeddings],
input_shape=(max_seq_length * 2, ),
trainable=False))
x.add(LSTM(50))
x.add(Dense(1, activation='sigmoid'))
x.compile(loss='mean_squared_error',
optimizer=tf.keras.optimizers.Adam(),
metrics=['accuracy'])
x.summary()
model = x
try:
# Start trainings
training_start_time = time()
callbacks = [EarlyStopping(monitor='val_loss', patience=3)]
malstm_trained = model.fit(X_train,
Y_train,
batch_size=batch_size,
def engine():
if request.method == 'GET':
algo = request.args.get('algo')
ptype = request.args.get('ptype')
tick = request.args.get('tick')
daysx = request.args.get('daysx')
if algo == "Delta":
algo = "adadelta"
elif algo == "Meta":
algo = "adam"
elif algo == "Gradient":
algo = "adagrad"
#importing the packages
#part 1
daysx = int(daysx)
import datetime as dt
import urllib.request, json
import pandas as pd #3
import numpy as np #3
# import matplotlib.pyplot as plt
# from matplotlib.pylab import rcParams
from sklearn.preprocessing import MinMaxScaler
#used for setting the output figure size
# rcParams['figure.figsize'] = 20,10
#to normalize the given input data
scaler = MinMaxScaler(feature_range=(0, 1))
#to read input data set (place the file name inside ' ') as shown below
ticker = tick
api_key = '3T9YAWICQG9J42JM'
url_string = "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=%s&outputsize=compact&apikey=%s" % (
ticker, api_key)
todataframe = pd.DataFrame()
with urllib.request.urlopen(url_string) as url:
datax = json.loads(url.read().decode())
datax = datax['Time Series (Daily)']
df = pd.DataFrame(
columns=['Date', 'Open', 'High', 'Low', 'Close', 'Volume'])
for k, v in datax.items():
date = dt.datetime.strptime(k, '%Y-%m-%d')
data_row = [
date.date(),
float(v['3. low']),
float(v['2. high']),
float(v['4. close']),
float(v['1. open']),
float(v['5. volume'])
]
# print(data_row)
df.loc[-1, :] = data_row
df.index = df.index + 1
todataframe = df
#todataframe.head()
#print(todataframe)
#part 2
#todataframe['Date'] = pd.to_datetime(todataframe.Date,format='%Y-%m-%d')
#todataframe.index = todataframe['Date']
#plt.figure(figsize=(16,8))
#plt.plot(todataframe['Close'], label='Closing Price')
#part 3
from sklearn.preprocessing import MinMaxScaler
from tensorflow.python.keras.layers import Dense, Dropout, LSTM
from tensorflow.python.keras import Sequential
#dataframe creation
seriesdata = todataframe.sort_index(ascending=True, axis=0)
new_seriesdata = pd.DataFrame(index=range(0, len(todataframe)),
columns=['Date', ptype])
length_of_data = len(seriesdata)
for i in range(0, length_of_data):
new_seriesdata['Date'][i] = seriesdata['Date'][i]
new_seriesdata[ptype][i] = seriesdata[ptype][i]
#setting the index again
new_seriesdata.index = new_seriesdata.Date
new_seriesdata.drop('Date', axis=1, inplace=True)
#creating train and test sets this comprises the entire data’s present in the dataset
myseriesdataset = new_seriesdata.values
totrain = myseriesdataset[0:new_seriesdata.size, :]
tovalid = myseriesdataset[new_seriesdata.size:, :]
#print(len(totrain))
#print(len(tovalid))
#part 4
scalerdata = MinMaxScaler(feature_range=(0, 1))
scale_data = scalerdata.fit_transform(myseriesdataset)
x_totrain, y_totrain = [], []
length_of_totrain = len(totrain)
for i in range(60, length_of_totrain):
x_totrain.append(scale_data[i - 60:i, 0])
y_totrain.append(scale_data[i, 0])
x_totrain, y_totrain = np.array(x_totrain), np.array(y_totrain)
x_totrain = np.reshape(x_totrain,
(x_totrain.shape[0], x_totrain.shape[1], 1))
#LSTM neural network
lstm_model = Sequential()
lstm_model.add(
LSTM(units=50,
return_sequences=True,
input_shape=(x_totrain.shape[1], 1)))
lstm_model.add(LSTM(units=50))
lstm_model.add(Dense(1))
lstm_model.compile(loss='mean_squared_error', optimizer=algo)
lstm_model.fit(x_totrain, y_totrain, epochs=3, batch_size=1, verbose=2)
#predicting next data stock price
myinputs = new_seriesdata[len(new_seriesdata) -
(len(tovalid) + daysx) - 60:].values
myinputs = myinputs.reshape(-1, 1)
myinputs = scalerdata.transform(myinputs)
tostore_test_result = []
for i in range(60, myinputs.shape[0]):
tostore_test_result.append(myinputs[i - 60:i, 0])
tostore_test_result = np.array(tostore_test_result)
tostore_test_result = np.reshape(
tostore_test_result,
(tostore_test_result.shape[0], tostore_test_result.shape[1], 1))
myclosing_priceresult = lstm_model.predict(tostore_test_result)
myclosing_priceresult = scalerdata.inverse_transform(
myclosing_priceresult)
#part 5
#print(len(tostore_test_result));
# print(myclosing_priceresult);
xm = myclosing_priceresult.tolist()
return jsonify(xm)
def make_model(self, embedding_matrics, max_tokens, X_train, X_test,
Y_train, Y_test):
"""
@ set up model
:param embedding_matrics:
:param max_tokens:
:param X_train:
:param X_test:
:param Y_train:
:param Y_test:
:return:
"""
model = Sequential()
# embedding层的作用其实就是查表，从而将高维稀疏矩阵转变为低纬稠密矩阵
model.add(
Embedding(TCconfig.num_words,
TCconfig.embedding_dim,
weights=[embedding_matrics],
input_length=max_tokens,
trainable=False))
# model.add(Bidirectional(LSTM(units=128, return_sequences=True)))
# model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(units=64, return_sequences=True)))
model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(units=32, return_sequences=False)))
model.summary()
print('Y_train.shape[1]:', Y_train.shape[1])
model.add(Dense(Y_train.shape[1], activation='softmax'))
optimizer = Adam(lr=1e-3)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
# defin callbacks
path_checkpoint = os.path.join(
envipath, 'Checkpoints/news_classifier_checkpoint.h5')
checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=False,
save_best_only=True)
earlystopping = EarlyStopping(monitor='val_loss',
patience=3,
verbose=1)
lr_reduction = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-5,
patience=0,
verbose=1)
tensorboard = TensorBoard(log_dir=os.path.join(envipath, 'Log'))
callbacks = [earlystopping, lr_reduction, checkpoint, tensorboard]
print('start to train........')
logger.info('start to train........')
# train
model.fit(X_train,
Y_train,
validation_split=0.1,
epochs=20,
batch_size=64,
shuffle=True,
callbacks=callbacks)
# print('Y_test:',Y_test)
Y_test = to_categorical(Y_test,
num_classes=self.classnames.values().__len__())
result = model.evaluate(X_test, Y_test)
logger.info('accuracy:{}'.format(result[1]))
print('accuracy:{}'.format(result[1]))
# save model
print('saving model is done...')
logger.info('saving model is done')
"""
Simple model with one temporal image input and one temporal tensor output
"""
from tensorflow.python.keras import Model
from tensorflow.python.keras.layers import Input, Dense, Concatenate, Flatten, TimeDistributed, LSTM
from export import export_current_model
input = Input(shape=(None, 64, 64, 1))
x = TimeDistributed(Flatten())(input)
x = TimeDistributed(Dense(10))(x)
x = LSTM(10, return_sequences=True)(x)
model = Model(inputs=[input], outputs=[x])
print(model.summary())
output_name = model.output.name.split(':')[0]
print('Name of output node:', output_name)
export_current_model('models/temporal_input_temporal_output.pb', output_name)
def __init__(self, units=1,
name=None,
rnn_type='SimpleRNN',
activation=linear,
kernel_initializer=default_kernel_initializer(),
recurrent_initializer=default_kernel_initializer(),
bias_initializer=default_bias_initializer(),
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
trainable=True,
dtype=None,):
if not dtype:
dtype = floatx()
elif not dtype == floatx():
set_floatx(dtype)
assert isinstance(name, str), \
"Please provide a string for field name. "
assert callable(activation), \
"Please provide a function handle for the activation. "
# prepare initializers.
if isinstance(kernel_initializer, (float, int)):
kernel_initializer = default_constant_initializer(kernel_initializer)
if isinstance(bias_initializer, (float, int)):
bias_initializer = default_constant_initializer(bias_initializer)
# prepare regularizers.
kernel_regularizer = default_regularizer(kernel_regularizer)
bias_regularizer = default_regularizer(bias_regularizer)
if rnn_type == 'SimpleRNN':
super(RNNField, self).__init__(
SimpleRNN(
units=units,
activation=activation,
return_sequences=True,
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
recurrent_regularizer=recurrent_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=name
)
)
elif rnn_type == 'LSTM':
super(RNNField, self).__init__(
LSTM(
units=units,
activation=activation,
return_sequences=True,
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
recurrent_regularizer=recurrent_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
unroll=True,
name=name
)
)
elif rnn_type == 'Dense':
super(RNNField, self).__init__(
Dense(
units=units,
activation=activation,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
kernel_regularizer=kernel_regularizer,
bias_regularizer=bias_regularizer,
trainable=trainable,
dtype=dtype,
name=name
)
)
else:
raise NotImplementedError('Supported RNNType: (SimpleRNN, LSTM, Dense)')
break
trainy = np.append(trainy, temp)
temp = np.zeros((38))
trainF.close()
np.savetxt('trainX.csv', trainX, delimiter=',')
np.savetxt('trainy.csv', trainy, delimiter=',')
# trainX = np.loadtxt('trainX.csv', delimiter=',')
# trainy = np.loadtxt('trainy.csv', delimiter=',')
# testX = np.loadtxt('testX.csv', delimiter=',')
# testy = np.loadtxt('testy.csv', delimiter=',')
optimizer = tf.keras.optimizers.SGD(lr=1e-12, momentum=0.1)
model = Sequential()
model.add(LSTM(128, return_sequences=True, input_shape=(None, 128)))
model.add(LSTM(30, input_shape=(128, 1)))
model.add(Dense(10))
model.add(Dense(38, activation='softmax'))
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=[
tf.metrics.MeanSquaredError(name='my_mse'),
tf.metrics.AUC(name='my_auc'),
])
trainX = trainX.reshape(trainX.shape[0], 1, trainX.shape[1])
testX = testX.reshape(testX.shape[0], 1, testX.shape[1])
trainX = np.array(trainX).transpose()
testX = np.array(testX).transpose()
trainy = trainy.reshape(148572, 38)
from tensorflow.python.keras.layers import LSTM
data = [[i for i in range(100)]]
data = np.array(data, dtype=float)
target = [[i for i in range(1, 101)]]
target = np.array(target, dtype=float)
data = data.reshape((1, 1, 100))
target = target.reshape((1, 1, 100))
x_test = [i for i in range(100, 200)]
x_test = np.array(x_test).reshape((1, 1, 100))
y_test = [i for i in range(101, 201)]
y_test = np.array(y_test).reshape(1, 1, 100)
model = Sequential()
model.add(LSTM(100, input_shape=(1, 100), return_sequences=True))
model.add(Dense(100))
model.compile(loss='mean_absolute_error',
optimizer='adam',
metrics=['accuracy'])
model.fit(data,
target,
nb_epoch=10000,
batch_size=1,
verbose=2,
validation_data=(x_test, y_test))
predict = model.predict(data)
print(predict)
X_train = np.reshape(np.asarray(X_train), (1, np.asarray(X_train).shape[0], np.asarray(X_train).shape[1]))
y_train = np.reshape(np.asarray(y_train), (1, np.asarray(y_train).shape[0], np.asarray(y_train).shape[1]))
"""Creating the LSTM Model and importing the required libraries"""
from tensorflow.python.keras import Sequential
from tensorflow import keras
from tensorflow.python.keras.layers import LSTM, Dropout, Dense, Conv2D, MaxPooling2D, Flatten,TimeDistributed
from keras.optimizers import Adam, SGD
"""Adam Optimizer is used and the loss function used here is Mean Sqaured error."""
model = Sequential()
model.add(LSTM(units=64, return_sequences = True, input_shape=(None, np.asarray(X_train).shape[2]), name="LSTM1"))
model.add(LSTM(units=64, return_sequences = True, name="LSTM2"))
model.add(LSTM(units=32, return_sequences = True, name="LSTM3"))
model.add(LSTM(units=32, return_sequences = True, name="LSTM4"))
model.add(LSTM(units=16, name="LSTM5"))
model.add(Dense(units=6, name="Dense1"))
opt = keras.optimizers.Adam(learning_rate=0.0001)
model.compile(optimizer=opt,loss= 'mean_squared_error')
hist = model.fit(X_train,y_train,epochs=100,batch_size=32)
model.summary()
plt.plot(hist.history['loss'])
plt.title('Model Loss')
plt.ylabel('Traning Loss')
def siamese_model(input_shape=None,
track_length=1,
features=None,
neighborhood_scale_size=10,
reg=1e-5,
init='he_normal',
softmax=True,
norm_method='std',
filter_size=61):
def compute_input_shape(feature):
if feature == 'appearance':
return input_shape
elif feature == 'distance':
return (None, 2)
elif feature == 'neighborhood':
return (None, 2 * neighborhood_scale_size + 1,
2 * neighborhood_scale_size + 1, 1)
elif feature == 'regionprop':
return (None, 3)
else:
raise ValueError('siamese_model.compute_input_shape: '
'Unknown feature `{}`'.format(feature))
def compute_reshape(feature):
if feature == 'appearance':
return (64, )
elif feature == 'distance':
return (2, )
elif feature == 'neighborhood':
return (64, )
elif feature == 'regionprop':
return (3, )
else:
raise ValueError('siamese_model.compute_output_shape: '
'Unknown feature `{}`'.format(feature))
def compute_feature_extractor(feature, shape):
if feature == 'appearance':
# This should not stay: channels_first/last should be used to
# dictate size (1 works for either right now)
N_layers = np.int(np.floor(np.log2(input_shape[1])))
feature_extractor = Sequential()
feature_extractor.add(InputLayer(input_shape=shape))
# feature_extractor.add(ImageNormalization2D('std', filter_size=32))
for layer in range(N_layers):
feature_extractor.add(
Conv3D(64, (1, 3, 3),
kernel_initializer=init,
padding='same',
kernel_regularizer=l2(reg)))
feature_extractor.add(BatchNormalization(axis=channel_axis))
feature_extractor.add(Activation('relu'))
feature_extractor.add(MaxPool3D(pool_size=(1, 2, 2)))
feature_extractor.add(Reshape((-1, 64)))
return feature_extractor
elif feature == 'distance':
return None
elif feature == 'neighborhood':
N_layers_og = np.int(
np.floor(np.log2(2 * neighborhood_scale_size + 1)))
feature_extractor_neighborhood = Sequential()
feature_extractor_neighborhood.add(
InputLayer(input_shape=(None, 2 * neighborhood_scale_size + 1,
2 * neighborhood_scale_size + 1, 1)))
for layer in range(N_layers_og):
feature_extractor_neighborhood.add(
Conv3D(64, (1, 3, 3),
kernel_initializer=init,
padding='same',
kernel_regularizer=l2(reg)))
feature_extractor_neighborhood.add(
BatchNormalization(axis=channel_axis))
feature_extractor_neighborhood.add(Activation('relu'))
feature_extractor_neighborhood.add(
MaxPool3D(pool_size=(1, 2, 2)))
feature_extractor_neighborhood.add(Reshape((-1, 64)))
return feature_extractor_neighborhood
elif feature == 'regionprop':
return None
else:
raise ValueError('siamese_model.compute_feature_extractor: '
'Unknown feature `{}`'.format(feature))
if features is None:
raise ValueError('siamese_model: No features specified.')
if K.image_data_format() == 'channels_first':
channel_axis = 1
input_shape = tuple([input_shape[0], None] + list(input_shape[1:]))
else:
channel_axis = -1
input_shape = tuple([None] + list(input_shape))
features = sorted(features)
inputs = []
outputs = []
for feature in features:
in_shape = compute_input_shape(feature)
re_shape = compute_reshape(feature)
feature_extractor = compute_feature_extractor(feature, in_shape)
layer_1 = Input(shape=in_shape)
layer_2 = Input(shape=in_shape)
inputs.extend([layer_1, layer_2])
# apply feature_extractor if it exists
if feature_extractor is not None:
layer_1 = feature_extractor(layer_1)
layer_2 = feature_extractor(layer_2)
# LSTM on 'left' side of network since that side takes in stacks of features
layer_1 = LSTM(64)(layer_1)
layer_2 = Reshape(re_shape)(layer_2)
outputs.append([layer_1, layer_2])
dense_merged = []
for layer_1, layer_2 in outputs:
merge = Concatenate(axis=channel_axis)([layer_1, layer_2])
dense_merge = Dense(128)(merge)
bn_merge = BatchNormalization(axis=channel_axis)(dense_merge)
dense_relu = Activation('relu')(bn_merge)
dense_merged.append(dense_relu)
# Concatenate outputs from both instances
merged_outputs = Concatenate(axis=channel_axis)(dense_merged)
# Add dense layers
dense1 = Dense(128)(merged_outputs)
bn1 = BatchNormalization(axis=channel_axis)(dense1)
relu1 = Activation('relu')(bn1)
dense2 = Dense(128)(relu1)
bn2 = BatchNormalization(axis=channel_axis)(dense2)
relu2 = Activation('relu')(bn2)
dense3 = Dense(3, activation='softmax')(relu2)
# Instantiate model
final_layer = dense3
model = Model(inputs=inputs, outputs=final_layer)
return model
