def ctpn(base_features, num_anchors, rnn_units=128, fc_units=512):
"""
ctpn网络
:param base_features: (B,H,W,C)
:param num_anchors: anchors个数
:param rnn_units:
:param fc_units:
:return:
"""
x = layers.Conv2D(512, kernel_size=(3, 3), padding='same', name='pre_fc')(base_features) # [B,H,W,512]
# 沿着宽度方式做rnn
rnn_forward = layers.TimeDistributed(layers.GRU(rnn_units, return_sequences=True, kernel_initializer='he_normal'),
name='gru_forward')(x)
rnn_backward = layers.TimeDistributed(
layers.GRU(rnn_units, return_sequences=True, kernel_initializer='he_normal', go_backwards=True),
name='gru_backward')(x)
rnn_output = layers.Concatenate(name='gru_concat')([rnn_forward, rnn_backward]) # (B,H,W,256)
# conv实现fc
fc_output = layers.Conv2D(fc_units, kernel_size=(1, 1), activation='relu', name='fc_output')(
rnn_output) # (B,H,W,512)
# 分类
class_logits = layers.Conv2D(2 * num_anchors, kernel_size=(1, 1), name='cls')(fc_output)
class_logits = layers.Reshape(target_shape=(-1, 2), name='cls_reshape')(class_logits)
# 中心点垂直坐标和高度回归
predict_deltas = layers.Conv2D(2 * num_anchors, kernel_size=(1, 1), name='deltas')(fc_output)
predict_deltas = layers.Reshape(target_shape=(-1, 2), name='deltas_reshape')(predict_deltas)
# 侧边精调(只需要预测x偏移即可)
predict_side_deltas = layers.Conv2D(num_anchors, kernel_size=(1, 1), name='side_deltas')(fc_output)
predict_side_deltas = layers.Reshape(target_shape=(-1, 1), name='side_deltas_reshape')(
predict_side_deltas)
return class_logits, predict_deltas, predict_side_deltas
def cnn_2x_encdec_siamese(voc_size, max_len, dropout=0.5):
"""Two siamese branches, each embedding a statement.
Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
Returns:
A Keras model instance.
"""
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.GRU(256)(x) #this acts as the encoder
x = layers.RepeatVector(256)(
x) #take the last output of GRU and feed it to the decoder
embedded_pivot = layers.GRU(256)(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='tanh')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model([pivot_input, statement_input], prediction)
return model
def main():
float_data = get_data('jena_climate_2009_2016.csv')
lookback = 1440
step = 6
delay = 144
batch_size = 128
data = normalize(float_data[:200000])
train_gen = generator(data, lookback=lookback, delay=delay, min_index=None, max_index=200000, step=step, batch_size=batch_size)
val_gen = generator(data, lookback=lookback, delay=delay, min_index=200001, max_index=300000, step=step, batch_size=batch_size)
test_gen = generator(data, lookback=lookback, delay=delay, min_index=300001, max_index=None, step=step, batch_size=batch_size)
model = Sequential()
model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5, return_sequences=True, input_shape=(None, data.shape[-1])))
model.add(layers.GRU(64, activation='relu', dropout=0.1, recurrent_dropout=0.5))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
val_steps = (300000 - 200001 - lookback)
history = model.fit_generator(train_gen, steps_per_epoch=500, epochs=40, validation_data=val_gen, validation_steps=val_steps, callbacks=model_callbacks())
epochs = range(1, len(loss) + 1)
loss = model_history.history['loss']
val_loss = model_history.history['val_loss']
plt.figure()
plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.show()
def build_decoder(self):
"""
Build a decoder that transforms latent representations into decoded outputs.
"""
self.latent_inputs = Input(shape=(self.latent_dim, ),
name='Vhod_v_dekodirnik')
x = layers.Dense(self.latent_dim,
activation='tanh',
name="Polno_povezan_sloj_2")(self.latent_inputs)
x = layers.Dropout(self.dropout_rate_mid, name="Izpustveni_sloj_2")(x)
x = layers.BatchNormalization(axis=-1,
name="Paketna_normalizacija_5")(x)
x = layers.RepeatVector(self.compound_length, name="Ponovi_vektor")(x)
x = layers.GRU(self.gru_size,
activation='tanh',
return_sequences=True,
name="GRU_sloj_1")(x)
x = layers.GRU(self.gru_size,
activation='tanh',
return_sequences=True,
name="GRU_sloj_2")(x)
x = layers.GRU(self.gru_size,
activation='tanh',
return_sequences=True,
name="GRU_sloj_3")(x)
x = layers.GRU(self.charset_length,
return_sequences=True,
activation='softmax')(x)
self.outputs = x
def transfer_deeprnn(models: List[Model] = None) -> Model:
# acc(train/valid/test): 0.87/0.88/0.862 | 2 epochs, commit ad1e | Adam lr 0.001
# Pre: models[0] is birnn
birnn = BaseModels.birnn() if models is None else models[0]
for i in range(4):
birnn.layers.pop()
X = birnn.layers[-1].output
for layer in birnn.layers:
layer.trainable = False
X = layers.Dropout(.5)(X)
X = layers.Bidirectional(
layers.GRU(units=128,
dropout=.2,
recurrent_dropout=.2,
return_sequences=True))(X)
X = layers.Bidirectional(
layers.GRU(units=128, dropout=.2, recurrent_dropout=.2))(X)
X = layers.Dropout(.5)(X)
X = layers.Dense(64, activation='relu')(X)
X = layers.Dense(1, activation='sigmoid')(X)
return Model(inputs=birnn.inputs,
outputs=X,
name=_name_model('transfer-deeprnn'))
def build_RNN(input_shape, dropout=0.5, recurrent_dropout=0.5,
stacked=False, bidirectional=False):
try:
assert isinstance(input_shape, tuple)
assert isinstance(dropout, (int, float)) and 0 <= dropout < 1
assert isinstance(recurrent_dropout, (int, float)) and 0 <= recurrent_dropout < 1
assert isinstance(stacked, (bool, int))
assert isinstance(bidirectional, (bool, int))
except AssertionError:
raise
model = models.Sequential()
if bidirectional:
model.add(layers.Bidirectional(
layers.GRU(32, input_shape=(None, input_shape[-1]))))
model.add(layers.Dense(1))
else:
model.add(layers.GRU(32, dropout=dropout,
recurrent_dropout=recurrent_dropout,
input_shape=(None, input_shape[-1]),
return_sequences=True if stacked else False))
if stacked:
model.add(layers.GRU(64, activation='relu', dropout=dropout,
recurrent_dropout=recurrent_dropout))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(lr=0.001), loss='mae')
return model
def build_text_rcnn(max_words=20000, maxlen=200, embedding_dim=400, classification_type=2):
sentence_input = layers.Input(shape=(maxlen,), dtype='int32')
embedded_sequences = layers.Embedding(max_words, embedding_dim)(sentence_input)
x_backwords = layers.GRU(100, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=True)(embedded_sequences)
x_backwords_reverse = layers.Lambda(lambda x: K.reverse(x, axes=1))(x_backwords)
x_fordwords = layers.GRU(100, return_sequences=True, kernel_regularizer=regularizers.l2(0.32 * 0.1), recurrent_regularizer=regularizers.l2(0.32), go_backwards=False)(embedded_sequences)
x_feb = layers.Concatenate(axis=2)([x_fordwords, embedded_sequences, x_backwords_reverse])
x_feb = layers.Dropout(0.32)(x_feb)
# Concatenate后的embedding_size
dim_2 = K.int_shape(x_feb)[2]
x_feb_reshape = layers.Reshape((maxlen, dim_2, 1))(x_feb)
filters = [2, 3, 4, 5]
conv_pools = []
for filter in filters:
conv =layers. Conv2D(filters=300,
kernel_size=(filter, dim_2),
padding='valid',
kernel_initializer='normal',
activation='relu',
)(x_feb_reshape)
pooled = layers.MaxPooling2D(pool_size=(maxlen - filter + 1, 1),
strides=(1, 1),
padding='valid',
)(conv)
conv_pools.append(pooled)
x = layers.Concatenate()(conv_pools)
x = layers.Flatten()(x)
output = layers.Dense(classification_type, activation='softmax')(x)
model = models.Model(sentence_input, output)
return model
def model_GRU(units_1st_layer,
input_shape,
num_outputs,
dropout_1st_layer,
recurrent_dropout_1st_layer,
metrics,
units_2nd_layer=None,
droput_2nd_layer=0.0,
recurrent_dropout_2nd_layer=0.0):
model = Sequential()
if units_2nd_layer:
model.add(
layers.GRU(units_1st_layer,
dropout=dropout_1st_layer,
recurrent_dropout=recurrent_dropout_1st_layer,
return_sequences=True,
input_shape=input_shape))
model.add(
layers.GRU(units_2nd_layer,
activation='relu',
dropout=droput_2nd_layer,
recurrent_dropout=recurrent_dropout_2nd_layer))
else:
model.add(
layers.GRU(units_1st_layer,
dropout=dropout_1st_layer,
recurrent_dropout=recurrent_dropout_1st_layer,
input_shape=input_shape))
model.add(layers.Dense(num_outputs, activation='sigmoid'))
model.compile(optimizer=RMSprop(),
loss='binary_crossentropy',
metrics=metrics)
return model
def model_GRU():
model = Sequential()
model.add(
layers.GRU(64,
dropout=0.1,
recurrent_dropout=0.6,
return_sequences=True,
input_shape=(600, gene_data.shape[-1])))
# model.add(layers.Dense(64, activation='relu'))
model.add(
layers.GRU(32,
dropout=0.1,
recurrent_dropout=0.5,
return_sequences=True,
input_shape=(600, gene_data.shape[-1])))
# model.add(layers.Dense(32, activation='relu'))
model.add(
layers.GRU(16,
dropout=0.1,
recurrent_dropout=0.4,
return_sequences=True,
input_shape=(32, gene_data.shape[-1])))
model.add(layers.Flatten())
model.add(layers.Dense(8, activation='relu'))
# model.add(layers.Flatten())
# model.add(layers.Dense(32, activation='relu'))
# model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
# model.add(Activation('sigmoid'))
return model
def gru_meatod_dropout_2(float_data, train_gen, val_gen, val_steps):
model = models.Sequential()
model.add(
layers.GRU(
32,
input_shape=(None, float_data.shape[-1]),
dropout=0.1,
recurrent_dropout=
0.3, # dropout，指定该层输入单元的dropout比率，recurrent_dropout指定循环单元的dropout比率
return_sequences=True))
model.add(
layers.GRU(
32,
input_shape=(None, float_data.shape[-1]),
dropout=0.1,
recurrent_dropout=
0.3, # dropout，指定该层输入单元的dropout比率，recurrent_dropout指定循环单元的dropout比率
return_sequences=True))
model.add(
layers.GRU(64, dropout=0.1, recurrent_dropout=0.3, activation='relu'))
model.add(layers.Dense(32))
model.add(layers.Dense(1))
model.compile(loss='mae', optimizer=optimizers.RMSprop())
history = model.fit_generator(
train_gen, # 数据生产器
steps_per_epoch=500, # 每轮抽取多少批次的生成器的数据，每批次128，共200000，
epochs=20, # 训练轮次
validation_data=val_gen, # 验证集，可以是numpy数组组成的元祖，也可以是数据生成器
validation_steps=val_steps # 从验证集中抽取多少个批次用于评估
)
return history
def build_neural_net(hyperparameters):
# the size of the embedding vector
embedding_vector_size: int = hyperparameters[2]
# whether to use a bidirectional RNN over the embedding
use_bidirectional: bool = hyperparameters[3]
# the vector size representing the input sequence
input_vector_representation_size: int = hyperparameters[4]
# whether to use a SimpleRNN, LSTM or GRU
use_SRNN_LSTM_GRU: str = hyperparameters[5]
# whether to use a second RNN after the first
use_second_RNN: bool = hyperparameters[6]
# the vector size of the second RNN
intermediate_vector_representation_size: int = hyperparameters[7]
# rate of dropout to use - float between 0 and 1
dropout: float = hyperparameters[8]
model = models.Sequential()
# Add an Embedding layer to the model, with as many inputs as terms in the vocab,
# and as many nodes as defined by the embedding_vector_size hyper parameter
model.add(layers.Embedding(len(vocab), embedding_vector_size, input_length=None, mask_zero=True))
# Add the first RNN Layer. If the use_bidirectional hyper parameter is set to True,
# then use a bidirectional implementation
if use_bidirectional:
# Add the first RNN Layer as a Simple RNN, LSTM or GRU depending on the use_SRNN_LSTM_GRU hyper parameter
# also use dropoput according to the hyper parameter
# and return sequences of the first layer depending on whether a second Recursive Layer will be used
if use_SRNN_LSTM_GRU == 'SRNN':
model.add(layers.Bidirectional(layers.SimpleRNN(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN)))
elif use_SRNN_LSTM_GRU == 'LSTM':
model.add(layers.Bidirectional(layers.LSTM(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN)))
elif use_SRNN_LSTM_GRU == 'GRU':
model.add(layers.Bidirectional(layers.GRU(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN)))
else:
if use_SRNN_LSTM_GRU == 'SRNN':
model.add(layers.SimpleRNN(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN))
elif use_SRNN_LSTM_GRU == 'LSTM':
model.add(layers.LSTM(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN))
elif use_SRNN_LSTM_GRU == 'GRU':
model.add(layers.GRU(input_vector_representation_size, dropout=dropout,
return_sequences=use_second_RNN))
if use_second_RNN:
if use_SRNN_LSTM_GRU == 'SRNN':
model.add(layers.SimpleRNN(intermediate_vector_representation_size, dropout=dropout))
elif use_SRNN_LSTM_GRU == 'LSTM':
model.add(layers.LSTM(intermediate_vector_representation_size, dropout=dropout))
elif use_SRNN_LSTM_GRU == 'GRU':
model.add(layers.GRU(intermediate_vector_representation_size, dropout=dropout))
# softmax layer
model.add(layers.Dense(19, activation='softmax'))
return model
def model_3(self):
embedding_matrix = self.build_myself_embedding_matrix()
input = layers.Input(shape=(self.max_words,))
embedding = layers.Embedding(input_dim=embedding_matrix.shape[0], output_dim=embedding_matrix.shape[1],
input_length=self.max_words, weights=[embedding_matrix])
x = layers.SpatialDropout1D(0.2)(embedding(input))
x = layers.Bidirectional(layers.GRU(400, return_sequences=True))(x)
x = layers.Bidirectional(layers.GRU(400, return_sequences=True))(x)
avg_pool = layers.GlobalAveragePooling1D()(x)
max_pool = layers.GlobalMaxPool1D()(x)
concat = layers.concatenate([avg_pool, max_pool])
x = layers.Dense(1024)(concat)
x = layers.BatchNormalization()(x)
x = layers.Activation(activation='relu')(x)
x = layers.Dropout(0.2)(x)
x = layers.Dense(512)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation(activation='relu')(x)
x = layers.Dropout(0.2)(x)
output = layers.Dense(self.class_num, activation='softmax')(x)
model = models.Model(input=input, output=output)
print(model.summary())
return model
def build_model(shape, vocab_size):
"""构建模型
"""
input_layer = layers.Input(shape=shape)
m = input_layer
m = layers.Embedding(vocab_size, 64)(m)
m = layers.Dropout(0.1)(m)
m = layers.GRU(
32,
return_sequences=True,
# recurrent_dropout=0.2,
kernel_regularizer=regularizers.l2(0.001))(m)
m = layers.GRU(
32,
return_sequences=True,
# recurrent_dropout=0.2,
kernel_regularizer=regularizers.l2(0.001))(m)
atten = m
atten = layers.Flatten()(atten)
atten = layers.Dense(shape[0], activation='softmax')(atten)
atten = layers.RepeatVector(32)(atten)
atten = layers.Permute((2, 1))(atten)
m = layers.Multiply()([m, atten])
# m = layers.Add()([m, emb])
m = layers.Flatten()(m)
m = layers.GaussianNoise(0.01)(m)
m = layers.Dense(300,
activation='linear',
kernel_regularizer=regularizers.l2(0.01))(m)
m = layers.BatchNormalization()(m)
m = layers.Activation('tanh')(m)
m = layers.Dropout(0.4)(m)
m = layers.Dense(300,
activation='linear',
kernel_regularizer=regularizers.l2(0.01))(m)
m = layers.BatchNormalization()(m)
m = layers.Activation('tanh')(m)
m = layers.Dropout(0.4)(m)
m = layers.Dense(len(LABEL_DICT), activation='softmax')(m)
atten_model = models.Model(inputs=[input_layer], outputs=atten)
model = models.Model(inputs=[input_layer], outputs=m)
optimizer = optimizers.Adam(lr=0.001, clipnorm=5.)
model.compile(optimizer, 'categorical_crossentropy', metrics=['accuracy'])
model.summary()
return model, atten_model
def iter_gru_model():
"""使用Dropout正则化的GRU堆叠层"""
model = models.Sequential()
model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5,
return_sequences=True, input_shape=(None, float_data.shape[-1])))
model.add(layers.GRU(32, dropout=0.2, recurrent_dropout=0.5))
model.add(layers.Dense(1))
return model
def build_model(GRU_SIZE=1024, WORDVEC_SIZE=300, ACTIVATION='relu'):
resnet = build_resnet()
# Global Image featuers (convnet output for the whole image)
input_img_global = layers.Input(shape=IMG_SHAPE)
image_global = resnet(input_img_global)
image_global = layers.BatchNormalization()(image_global)
image_global = layers.Dense(WORDVEC_SIZE / 2,
activation=ACTIVATION)(image_global)
image_global = layers.BatchNormalization()(image_global)
image_global = layers.RepeatVector(MAX_WORDS)(image_global)
# Local Image features (convnet output inside the bounding box)
input_img_local = layers.Input(shape=IMG_SHAPE)
image_local = resnet(input_img_local)
image_local = layers.BatchNormalization()(image_local)
image_local = layers.Dense(WORDVEC_SIZE / 2,
activation=ACTIVATION)(image_local)
image_local = layers.BatchNormalization()(image_local)
image_local = layers.RepeatVector(MAX_WORDS)(image_local)
# Context Vector input
# normalized to [0,1] the values:
# left, top, right, bottom, (box area / image area)
input_ctx = layers.Input(shape=(5, ))
ctx = layers.BatchNormalization()(input_ctx)
ctx = layers.RepeatVector(MAX_WORDS)(ctx)
language_model = models.Sequential()
input_words = layers.Input(shape=(MAX_WORDS, ), dtype='int32')
language = layers.Embedding(words.VOCABULARY_SIZE,
WORDVEC_SIZE,
input_length=MAX_WORDS)(input_words)
language = layers.BatchNormalization()(language)
language = layers.GRU(GRU_SIZE, return_sequences=True)(language)
language = layers.BatchNormalization()(language)
language = layers.TimeDistributed(
layers.Dense(WORDVEC_SIZE, activation=ACTIVATION))(language)
language = layers.BatchNormalization()(language)
# Problem with Keras 2:
# TypeError: Tensors in list passed to 'values' of 'ConcatV2' Op have types [uint8, uint8, bool, uint8] that don't all match.
# Masking doesn't work along with concatenation.
# How do I get mask_zero=True working in the embed layer?
x = layers.concatenate([image_global, image_local, ctx, language])
x = layers.GRU(GRU_SIZE)(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(words.VOCABULARY_SIZE, activation='softmax')(x)
return models.Model(
inputs=[input_img_global, input_img_local, input_words, input_ctx],
outputs=x)
def create_model(x, y, maxlen, epochs):
"""
Creates and trains a model.
:param x: Numpy boolean array of shape
(Number of sequences, maxlen, number of distinct character)
:type x: numpy.ndarray
:param y: Numpy boolean array of shape
(Number of sequences, number of distinct character)
:type y: numpy.ndarray
:param maxlen: the length of a sequence to extract as train
:type maxlen: int
:param epochs: number of training iterations
:type epochs: int
:param chars: list of unique characters
:type chars: list
:returns: trained keras model
:rtype: keras.engine.sequential.Sequential
"""
# model = Sequential()
# model.add(layers.GRU(
# 32,
# return_sequences=True,
# input_shape=(maxlen, len(chars)))
# )
# model.add(layers.GRU(
# 64,
# input_shape=(maxlen, len(chars)))
# )
# model.add(layers.Dense(
# len(chars),
# activation='softmax')
# )
# start of my model attempt, it works decently well
# - try commenting it out and using the previous one
# - also try removing the Dropout layers
# --------------------------------------------
model = Sequential()
model.add(
layers.GRU(256, return_sequences=True, input_shape=(maxlen, num_vals)))
model.add(layers.Dropout(0.5))
model.add(layers.GRU(128))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(
num_vals,
activation='softmax')) #alternatively: activation = 'softmax'
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
# --------------------------------------------
print(model.summary())
model.fit(x, y, batch_size=128, epochs=epochs)
return (model)
def create_gru_model():
model = Sequential()
model.add(layers.GRU(64,
dropout=0.1,
recurrent_dropout=0.4,
return_sequences=True,
input_shape=(None, scal.FEATURE_NUM)))
model.add(layers.GRU(128,activation='relu',dropout=0.1,recurrent_dropout=0.4))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer=RMSprop(),loss='binary_crossentropy')
return model
def _build_gru(self):
inp = KL.Input(shape=(100, self.feature_size))
x = inp
x = KL.Masking()(x)
x = KL.GRU(32, return_sequences=True)(x)
x = KL.GRU(1)(x)
out = x
model = keras.Model(inp, out)
model.compile(optimizer=keras.optimizers.Adam(lr=0.001), loss='mse')
return model
def model_jena_p218():
model = models.Sequential()
model.add(
layers.GRU(32,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, 7)))
model.add(layers.GRU(64, dropout=0.2, recurrent_dropout=0.2))
model.add(layers.Dense(1))
return model
def build_model(self,x_train,y_train,batch_size,epochs=20):
timesteps, dim = x_train.shape[1],x_train.shape[2]
self.model.add(layers.GRU(128,dropout=0.2,input_shape=(timesteps,dim),return_sequences=True))
self.model.add(layers.BatchNormalization())
self.model.add(layers.GRU(128,dropout=0.0,return_sequences=True))
self.model.add(layers.BatchNormalization())
self.model.add((layers.Dense(1)))
print(self.model.summary())
self.model.compile(optimizer='adam',loss='mae')
self.history=self.model.fit(x_train,y_train,batch_size=batch_size,validation_split=0.2,verbose=1,epochs=epochs)
self.history_dict=self.history.history
return self.model,self.history_dict
def train_point(self, number):
model = keras.Sequential()
model.add(
layers.GRU(units=100, use_bias=False, input_shape=(number+1, 4), return_sequences=True, name='GRU_1')
)
model.add(layers.Dropout(0.3))
model.add(
layers.GRU(100, use_bias=True, return_sequences=False, name='GRU_2')
)
model.add(layers.Dropout(0.3))
model.add(layers.Dense(units=1, use_bias=True, name='DENSE_1'))
model.summary()
model.compile(optimizer='adam',
loss='mse',
metrics=['accuracy'])
dataManager = DataManager('D:/DataSet/stock/preparation/')
train_x, train_y = dataManager.creatData_Method_2(number)
index = numpy.arange(len(train_x))
numpy.random.shuffle(index)
train_x = train_x[index]
train_y = train_y[index]
print('train epoch: ', len(train_x))
Log_dir = 'D:/Git/tensorflow_cpu/StockPractice/'
file_name = 'stock_GRU_GRU_DENSE'
tb = TensorBoard(log_dir=Log_dir + file_name, write_grads=True)
model.fit(train_x, train_y,
batch_size=256,
shuffle=True,
epochs=5,
verbose=1)
# callbacks=[tb],
#validation_data=[test_x,test_y])
model.save(filepath=Log_dir+'stock_GRU_GRU_DENSE_point.h5',overwrite=True)
testManager = DataManager('D:/DataSet/stock/test/')
test_x,test_y = testManager.creatData_Method_2(number)
print(' test epoch: ', len(test_x))
predict_y = model.predict_on_batch(test_x)
plt.plot(range(len(test_y)),test_y,'-')
plt.plot(range(len(test_y)), predict_y, '-')
plt.show()
def transfer_learn_ocr_model(img_shape, conv_filters, kernel_size, pool_size, time_dense_size, dropout, rnn_size,
act_conv,act_dense, n_classes, max_string_len, training=True):
# set up VGG19 covolutional layers
input_data = layers.Input(name='the_input', shape=img_shape)
vgg_model = VGG16(include_top=False, weights='imagenet', input_tensor=input_data)
for i in range(len(vgg_model.layers)):
vgg_model.layers[i].trainable = False
# get only 3 block of convolutional layers
vgg_last_layer = vgg_model.layers[9].output
# do recurent part
# do recurent part
conv_to_rnn_dims = (int(vgg_last_layer.shape[1]), int(vgg_last_layer.shape[2] * vgg_last_layer.shape[3]))
inner = layers.Reshape(target_shape=conv_to_rnn_dims, name='reshape')(vgg_last_layer)
# dropout1 = layers.Dropout(dropout, name='dropout1')(inner)
# cuts down input size going into RNN:
inner_dense = layers.Dense(time_dense_size, activation=act_dense, name='time_dense')(inner)
inner_dense = layers.BatchNormalization()(inner_dense)
# Two layers of bidirecitonal GRUs
# GRU seems to work as well, if not better than GRU:
gru_1 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1_a')(inner_dense)
gru_1b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal',
name='gru1_b')(inner_dense)
gru1_merged = layers.merge([gru_1, gru_1b], mode='sum')
gru1_merged = layers.BatchNormalization()(gru1_merged)
gru_2 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2_a')(gru1_merged)
gru_2b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
gru_merged = layers.merge([gru_2, gru_2b], mode='concat')
gru_merged = layers.BatchNormalization()(gru_merged)
# dropout2 = layers.Dropout(dropout, name='dropout2')(gru_merged)
# transforms RNN output to character activations: 0-9 + '/' + ' ' + null
dense = layers.Dense(n_classes, kernel_initializer='he_normal', name='class_dense')(gru_merged)
y_pred = layers.Activation('softmax', name='softmax')(dense)
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
if training:
# make layers for output
labels = layers.Input(name='the_labels', shape=[max_string_len], dtype='float64')
input_length = layers.Input(name='input_length', shape=[1], dtype='int64')
label_length = layers.Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')(
[y_pred, labels, input_length, label_length])
model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
else:
model = Model(input_data, y_pred)
model.summary()
return model
def train_model(gen_train, gen_valid, idx):
model = models.Sequential()
model.add(layers.InputLayer(input_shape=(None, 4)))
model.add(layers.Masking(mask_value=0., input_shape=(None, 4)))
model.add(layers.BatchNormalization())
model.add(
layers.Bidirectional(layers.GRU(16, return_sequences=True),
merge_mode='ave'))
model.add(layers.BatchNormalization())
model.add(
layers.Bidirectional(layers.GRU(16, return_sequences=True),
merge_mode='ave'))
model.add(layers.BatchNormalization())
model.add(layers.Dense(16))
model.add(layers.BatchNormalization())
model.add(layers.Activation('relu'))
model.add(layers.Dense(2, activation='sigmoid'))
model.summary()
callbacks_list = [
callbacks.EarlyStopping(monitor="val_symmetric_accuracy",
patience=1,
min_delta=0.001,
mode='max'),
callbacks.ModelCheckpoint(filepath="track_%i_weights.h5" % idx,
monitor="val_symmetric_accuracy",
save_best_only=True,
save_weights_only=True),
callbacks.ReduceLROnPlateau(monitor="val_symmetric_accuracy",
factor=0.5,
mode='max',
min_delta=0.001,
patience=1)
]
model.compile(optimizer=optimizers.Adam(lr=0.002),
loss=losses.binary_crossentropy,
metrics=[symmetric_accuracy, mask_accuracy])
out = model.fit_generator(gen_train,
steps_per_epoch=len(gen_train),
epochs=50,
callbacks=callbacks_list,
validation_data=gen_valid,
validation_steps=len(gen_valid))
model.save('model_tracker_%i.h5' % idx)
with open("history_track_%i.pkl" % idx, "wb") as file:
pickle.dump(out.history, file)
def GRU_vol(input_dim, time_step, batch_size, epochs, activation,
bias_initializer, kernel_initializer, bias_regularizer,
hidden_layers, dropout, dropout_rate, batch_normalization,
early_stop, x_train, y_train, lr):
model = Sequential()
model.add(
layers.GRU(hidden_layers[0],
input_shape=[time_step, input_dim],
activation=activation,
bias_initializer=bias_initializer,
kernel_initializer=kernel_initializer,
bias_regularizer=bias_regularizer,
return_sequences=True))
if batch_normalization:
model.add(BatchNormalization())
if dropout:
model.add(Dropout(dropout_rate))
for hidden_layer in hidden_layers[1:]:
model.add(
layers.GRU(hidden_layer,
activation=activation,
bias_initializer=bias_initializer,
kernel_initializer=kernel_initializer,
return_sequences=True))
if batch_normalization:
model.add(BatchNormalization())
if dropout:
model.add(Dropout(dropout_rate))
model.add(Dense(1))
model.compile(loss=keras.losses.mse,
optimizer=keras.optimizers.Adam(lr=lr))
if early_stop:
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
callbacks=[EarlyStopping(patience=10)],
validation_split=0.2)
else:
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0)
return model
def unit(x):
ident = x
x = layers.Bidirectional(
GRU(Var.hidden_units,
activation=Var.activationF,
return_sequences=True,
kernel_regularizer=regularizers.l2(Var.Kr),
activity_regularizer=regularizers.l2(Var.Ar),
kernel_constraint=max_norm(max_value=3.),
dropout=Var.dropout,
recurrent_dropout=Var.dropout))(x)
x = layers.GRU(Var.hidden_units * 2,
return_sequences=True,
kernel_regularizer=regularizers.l2(Var.Kr),
activity_regularizer=regularizers.l2(Var.Ar),
kernel_constraint=max_norm(max_value=3.),
dropout=Var.dropout,
recurrent_dropout=Var.dropout)(x)
x = layers.Bidirectional(
GRU(Var.hidden_units * 2,
return_sequences=True,
kernel_regularizer=regularizers.l2(Var.Kr),
activity_regularizer=regularizers.l2(Var.Ar),
kernel_constraint=max_norm(max_value=3.),
dropout=Var.dropout,
recurrent_dropout=Var.dropout))(x)
x = layers.GRU(Var.hidden_units * 4,
return_sequences=True,
kernel_regularizer=regularizers.l2(Var.Kr),
activity_regularizer=regularizers.l2(Var.Ar),
kernel_constraint=max_norm(max_value=3.),
dropout=Var.dropout,
recurrent_dropout=Var.dropout)(x)
x = layers.Bidirectional(
GRU(Var.hidden_units * 4,
return_sequences=True,
kernel_regularizer=regularizers.l2(Var.Kr),
activity_regularizer=regularizers.l2(Var.Ar),
kernel_constraint=max_norm(max_value=3.),
dropout=Var.dropout,
recurrent_dropout=Var.dropout))(x)
x = layers.Dropout(Var.dropout,
noise_shape=(None, 1, Var.hidden_units * 8))(x)
x = layers.Dense(Var.Dense_Unit,
activation=Var.activationF,
kernel_constraint=max_norm(max_value=3.))(x)
# x = merge([ident,x], mode = 'sum') #mode 'sum' concat
x = layers.add([ident, x])
return x
def buildGRUModel(input_size, seq_len, hidden_size):
model = km.Sequential()
model.add(
kl.GRU(input_dim=input_size,
output_dim=hidden_size,
return_sequences=False))
model.add(kl.Dense(hidden_size, activation="relu"))
model.add(kl.RepeatVector(seq_len))
model.add(kl.GRU(hidden_size, return_sequences=True))
model.add(
kl.TimeDistributed(kl.Dense(output_dim=input_size,
activation="linear")))
model.compile(loss="mse", optimizer='adam')
return model
def create_model():
# Define a model that utilizes bidirectional GRU/LSTM celss
model = keras.models.Sequential()
model.add(L.InputLayer([None], dtype='int32'))
model.add(L.Embedding(len(all_words), 64))
model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True)))
model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True)))
model.add(L.Bidirectional(L.GRU(64, dropout=0.5, return_sequences=True)))
# add top layer that predicts tag probabilities
model.add(L.TimeDistributed(L.Dense(len(all_tags), activation='softmax')))
return model
def train():
stims, resps = ld.load_data(['Noise 1','Long Square','Short Square'])
time_steps = stims.shape[1]
repeats = stims.shape[0]
model = km.Sequential([
kl.GRU(50, return_sequences=True, input_shape=(time_steps,1)),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.GRU(50, return_sequences=True),
kl.Dense(1)
])
model.compile(loss='mse',
optimizer='adam',
metrics=['accuracy'])
stims = stims.reshape(stims.shape[0], stims.shape[1], 1)
resps = resps.reshape(resps.shape[0], resps.shape[1], 1)
model.fit(stims, resps, epochs=1000, batch_size=10,
callbacks=[ kc.ModelCheckpoint('output/model_lstm.h5') ])
def vanilla_ocr_model(img_shape, conv_filters, kernel_size, pool_size, time_dense_size, dropout, rnn_size, act_conv,
act_dense, n_classes, max_string_len):
input_data = layers.Input(name='the_input', shape=img_shape)
inner = layers.Conv2D(conv_filters, kernel_size, padding='same', activation=act_conv,
name='conv1_1')(input_data)
inner = layers.MaxPooling2D(pool_size=pool_size, name='max1')(inner)
inner = layers.Conv2D(conv_filters, kernel_size, padding='same', activation=act_conv,
name='conv2_1')(inner)
inner = layers.MaxPooling2D(pool_size=pool_size, name='max2')(inner)
# reshape along conv and width
conv_to_rnn_dims = (int(inner.shape[1]), int(inner.shape[2] * inner.shape[3]))
inner = layers.Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
# dropout1 = layers.Dropout(dropout, name='dropout1')(inner)
# cuts down input size going into RNN:
inner_dense = layers.Dense(time_dense_size, activation=act_dense, name='time_dense')(inner)
inner_dense = layers.BatchNormalization()(inner_dense)
# Two layers of bidirecitonal GRUs
# GRU seems to work as well, if not better than GRU:
gru_1 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1_a')(inner_dense)
gru_1b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal',
name='gru1_b')(inner_dense)
gru1_merged = layers.merge([gru_1, gru_1b], mode='sum')
gru1_merged = layers.BatchNormalization()(gru1_merged)
gru_2 = layers.GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2_a')(gru1_merged)
gru_2b = layers.GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
gru_merged = layers.merge([gru_2, gru_2b], mode='concat')
gru_merged = layers.BatchNormalization()(gru_merged)
# dropout2 = layers.Dropout(dropout, name='dropout2')(gru_merged)
# transforms RNN output to character activations: 0-9 + '/' + ' ' + null
dense = layers.Dense(n_classes, kernel_initializer='he_normal', name='class_dense')(gru_merged)
y_pred = layers.Activation('softmax', name='softmax')(dense)
# show model
Model(inputs=input_data, outputs=y_pred).summary()
# make layers for output
labels = layers.Input(name='the_labels', shape=[max_string_len], dtype='float64')
input_length = layers.Input(name='input_length', shape=[1], dtype='int64')
label_length = layers.Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = layers.Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')(
[y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
# loss with CTC
model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
return model
def build_han(maxlen, max_sent_len, max_words, embedding_dim, classification_type):
embedding_layer = layers.Embedding(max_words, embedding_dim, input_length=maxlen)
sentence_input = layers.Input(shape=(maxlen,), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
l_lstm = layers.Bidirectional(layers.GRU(100, return_sequences=True))(embedded_sequences)
l_att = Word_Attention(100)(l_lstm)
sentEncoder = models.Model(sentence_input, l_att)
sentEncoder.summary()
review_input = layers.Input(shape=(max_sent_len, maxlen), dtype='int32')
review_encoder = layers.TimeDistributed(sentEncoder)(review_input)
l_lstm_sent = layers.Bidirectional(layers.GRU(100, return_sequences=True))(review_encoder)
l_att_sent = Word_Attention(100)(l_lstm_sent)
preds = layers.Dense(classification_type, activation='softmax')(l_att_sent)
model = models.Model(review_input, preds)
return model
