def RNNGenerator(dataset):
features = [
tf.compat.v2.feature_column.numeric_column(k, dtype=tf.dtypes.float64)
for k in dataset.columns.values if (k != 'real' and k != 'actual')
]
def customRELU(x):
return tf.keras.activations.relu(x, max_value=100)
model = tf.keras.models.Sequential()
model.add(layers.Reshape([len(features), 1]))
model.add(
layers.SimpleRNN(128,
return_sequences=True,
kernel_regularizer='l1',
bias_regularizer='l2',
activation='relu'))
model.add(
layers.SimpleRNN(128,
return_sequences=True,
kernel_regularizer='l1_l2',
bias_regularizer='l2'))
model.add(
layers.SimpleRNN(128,
return_sequences=False,
kernel_regularizer='l1_l2',
bias_regularizer='l2'))
model.add(layers.Dropout(.2))
model.add(layers.Dense(128))
model.add(layers.Dense(len(features), activation=customRELU))
return model
def __init__(self, units):
super(MyRNN, self).__init__()
# 词向量编码 [b, 80] => [b, 80, 100]
# 构建embedding层 总词数,词向量位数,单词数
# 参数顺序,与上面 [b, 80, 100] 不同
# 可以直接 self.embedding = layers.Embedding(total_words, embedding_len)
# 因为不是直接连着全连接层,所以不需要input_length=max_review_len 这个参数
self.embedding = layers.Embedding(total_words,
embedding_len,
input_length=max_review_len)
# 构建RNN
# [b, 80, 100] => [b, 64] 这里的64 就是参数中的 units,就是h的位数
self.rnn = keras.Sequential([
# 先跑完一层,在跑另一层,因此 return_sequences=True
layers.SimpleRNN(units, dropout=0.5, return_sequences=True),
layers.SimpleRNN(units, dropout=0.5)
])
# 构建分类网络，用于将rnn的输出特征进行分类，2分类
# [b, 64] => [b, 1]
self.outlayer = Sequential([
layers.Dense(32),
layers.Dropout(rate=0.5),
layers.ReLU(),
layers.Dense(1)
])
def __init__(self, units):
super(RNNUsingSimpleRNN, self).__init__()
# transform text to embedding representation
# [b, 80] => [b, 80, 100]
self.embedding = layers.Embedding(total_words,
embedding_len,
input_length=max_review_len)
# [b, 80, 100] , h_dim: 64
self.rnn = keras.Sequential([
layers.SimpleRNN(units,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(units,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(units,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(units, dropout=0.5, unroll=True)
])
# fc, [b, 80, 100] => [b, 64] => [b, 1]
self.output_layer = layers.Dense(1)
def RNNSimpleModel(units_layer1, units_layer2):
inputs = keras.Input(shape=[max_sentence_word_lenth])
# 经过embeding 从[batchsize,80] 到 [batchsize,80,50]
X = keras.layers.Embedding(total_words,
50,
input_length=max_sentence_word_lenth)(inputs)
print(X.shape)
# 经过一层RNN
X, test = layers.SimpleRNN(units_layer1,
return_sequences=True,
return_state=True,
dropout=0.2)(X)
print(X.shape)
print(test.shape)
X = layers.SimpleRNN(units_layer2, dropout=0.2)(X)
# 注意 这里 如果 两个return都设定True 那么两个返回的都是正常的张量
# 如果这里只设定return_state = True 那么这里返回的是一个列表
# 如果什么都不设定，那么就直接返回最后一个隐藏状态的张量
# print(len(X))
# print(X[0].shape,id(X[0]))
# print(X[1].shape,id(X[1]))
X = layers.Dense(1, activation='sigmoid')(X)
MyModel = keras.Model(inputs=inputs, outputs=X)
return MyModel
def rnn_sequential(model, model_name, h_dim, return_seq, name=None):
"""Add one rnn layer in sequential model.
Args:
- model: sequential rnn model
- model_name: rnn, lstm, or gru
- h_dim: hidden state dimensions
- return_seq: True or False
- name: layer name
Returns:
- model: sequential rnn model
"""
if name == None:
if model_name == "rnn":
model.add(layers.SimpleRNN(h_dim, return_sequences=return_seq))
elif model_name == "lstm":
model.add(layers.LSTM(h_dim, return_sequences=return_seq))
elif model_name == "gru":
model.add(layers.GRU(h_dim, return_sequences=return_seq))
else:
if model_name == "rnn":
model.add(
layers.SimpleRNN(h_dim, return_sequences=return_seq,
name=name))
elif model_name == "lstm":
model.add(
layers.LSTM(h_dim, return_sequences=return_seq, name=name))
elif model_name == "gru":
model.add(layers.GRU(h_dim, return_sequences=return_seq,
name=name))
return model
def __init__(self, units, embedding_len, input_len, total_words):
"""
:param units: simrnn 向量大小
:param embedding_len: 单词的向量长度
:param input_len: 输入每个单个时间序列的长度
:param total_words: 句子数
"""
super(RNN, self).__init__()
# embedding
self.embedding = layers.Embedding(input_dim=total_words,
output_dim=embedding_len,
input_length=input_len)
# RnnLayer
self.rnn = Sequential([
layers.SimpleRNN(units,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(units, dropout=0.5, unroll=True)
])
# fullConnection
self.fc = layers.Dense(1)
def only_simple_rnn():
model = models.Sequential([
layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),
layers.SimpleRNN(20, return_sequences=True),
layers.SimpleRNN(1)
])
return model
def simple_rnn(dense_node):
model = models.Sequential([
layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),
layers.SimpleRNN(20),
layers.Dense(dense_node)
])
return model
def make_model():
my_model = keras.Sequential()
my_model.add(layers.SimpleRNN(5, return_sequences=True))
my_model.add(layers.SimpleRNN(5))
my_model.add(layers.Dense(1, activation=tf.keras.activations.sigmoid))
my_model.compile(loss=tf.keras.losses.MeanAbsoluteError(),
optimizer=tf.keras.optimizers.Nadam(1e-4),
metrics=['mae'])
return my_model
def simple_rnn_2(dense_node):
model = models.Sequential([
layers.SimpleRNN(20, return_sequences=True, input_shape=[None, 1]),
layers.SimpleRNN(20, return_sequences=True),
# 모든 timestep에 Dense(10) 적용
layers.TimeDistributed(layers.Dense(dense_node))
])
return model
def __init__(self, units):
super(MyRnn, self).__init__()
self.embedding = layers.Embedding(total_words,
embedding_len,
input_length=max_review_len)
self.rnn = keras.Sequential([
layers.SimpleRNN(units, dropout=0.5, return_sequences=True, unroll=True),
layers.SimpleRNN(units, dropout=0.5, unroll=True)
])
self.outlayer = layers.Dense(1)
def RNN_model():
model = keras.Sequential([
layers.Embedding(input_dim=30000, output_dim=32, input_length=maxlen),
layers.SimpleRNN(32, return_sequences=True),
layers.SimpleRNN(1, activation='sigmoid', return_sequences=False)
])
model.compile(optimizer=keras.optimizers.Adam(),
loss=keras.losses.BinaryCrossentropy(),
metrics=['accuracy'])
return model
def define_NN(self):
tf.keras.backend.clear_session() # For easy reset of notebook state.
in1 = keras.Input(shape=(ImgGenerator.H, ImgGenerator.W // 2, 2), name='inp1')
in_r1 = layers.Reshape((ImgGenerator.H, ImgGenerator.W), name="reshaped_input")(in1)
lstm = layers.LSTM(units=256 * 4, name="lstm")(in_r1)
lstm = layers.Reshape((128, 8), name="reshaped_lstm")(lstm)
print(lstm)
in_conv = layers.DepthwiseConv2D((128, 1), padding="same", data_format='channels_last', name="depth-conv")(in1)
print(in_conv)
in_conv = layers.Reshape((128, 128), name="reshape_conv")(in_conv)
rnn = layers.SimpleRNN(256, name="rnn")(in_r1)
rnn = layers.Reshape((128, 2), name="reshaped_rnn")(rnn)
print(rnn)
# Rotated
rot_layer = layers.Lambda(lambda x: kbck.reverse(x, axes=0), output_shape=(64, 128, 2))(in1)
rot_layer = layers.Reshape((ImgGenerator.H, ImgGenerator.W), name="reshaped_input2")(rot_layer)
in_conv2 = layers.SimpleRNN(128, name="rnn2")(rot_layer)
print(in_conv2)
in_conv2 = layers.Reshape((128, 1), name="reshape_conv2")(in_conv2)
d0 = layers.Dense(1024, activation="tanh", name="dense-inp")(in_r1) #
print(d0)
d0 = layers.Concatenate(axis=2)([d0, lstm, in_conv, rnn, in_conv2])
print(d0.shape)
rnn = layers.Flatten()(d0)
# rnn = layers.BatchNormalization(momentum=0.8)(rnn)
# rnn = layers.LeakyReLU()(rnn)
print(rnn)
dense_1 = layers.Dense(2048, activation="relu")(rnn) # , activation="relu"
# dense_1 = layers.BatchNormalization(momentum=0.8)(dense_1)
# dense_1 = layers.LeakyReLU()(dense_1)
print(dense_1)
# for layer_idx in range(0, 5):
# dense_1 = layers.BatchNormalization(momentum=0.8)(dense_1)
# dense_1 = layers.Dense(1024, activation="tanh", name=f"muldence{layer_idx}")(dense_1)#, activation="relu"
dense_2 = layers.Dense(4096, activation="relu")(dense_1) # , activation="relu"
# dense_2 = layers.BatchNormalization(momentum=0.8)(dense_2)
print(dense_2)
output = layers.Dense(128 * 128)(dense_2) # ,, activation="softplus"
# output = layers.Softmax()(output)
print(f"Last dense:{output}")
output = layers.Reshape((128, 128))(output)
print(f"Out layer:{output}")
return [in1], [output]
def __init__(self, ):
super(MyRNN, self).__init__()
# [b, 8, 804] ,
self.rnn = keras.Sequential([
layers.SimpleRNN(521,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(804, dropout=0.5, unroll=True)
])
self.outlayer = layers.Dense(804)
def __init__(self, units):
super(MyRNN, self).__init__()
self.rnn = keras.Sequential([
layers.SimpleRNN(units, dropout=0.3, return_sequences=True),
layers.SimpleRNN(units, dropout=0.3)
])
self.outlayer = Sequential([
layers.Dense(128),
layers.Dropout(rate=0.5),
layers.ReLU(),
layers.Dense(1)])
def get_model_architecture2(n_classes, layers_config):
model = tf.keras.Sequential()
rnn_idx = [
i for i in range(len(layers_config))
if layers_config[i][0] == RNN_NAME or layers_config[i][0] == LSTM_NAME
]
input_shape = config.n_mels
for i, (name, dim) in enumerate(layers_config):
if name == RNN_NAME or name == LSTM_NAME:
# If last rnn layer
if i == rnn_idx[-1]:
return_sequence = False
else:
return_sequence = True
if i == 0:
if name == RNN_NAME:
model.add(
layers.SimpleRNN(dim,
input_shape=(None, input_shape),
activation='relu',
return_sequences=return_sequence))
else:
model.add(
layers.LSTM(dim,
input_shape=(None, input_shape),
return_sequences=return_sequence,
activation='relu',
recurrent_activation='relu'))
else:
if name == RNN_NAME:
model.add(
layers.SimpleRNN(dim,
activation='relu',
return_sequences=return_sequence))
else:
model.add(
layers.LSTM(dim, return_sequences=return_sequence))
elif name == FC_NAME:
if i == 0:
model.add(
layers.Dense(dim, activation='relu',
input_dim=input_shape))
else:
model.add(layers.Dense(dim, activation='relu'))
model.add(layers.Dense(n_classes, activation='softmax'))
return model
pass
def __init__(
self,
hidden_layers_sizes=(64, ),
activation='relu',
):
super().__init__()
self.regressor = None
self.hidden_layers_sizes = hidden_layers_sizes
self.rnn = []
for i, n_units in enumerate(self.hidden_layers_sizes):
self.rnn.append(layers.SimpleRNN(units=n_units))
self.rnn = layers.SimpleRNN(units=64)
self.dense = layers.Dense(1)
def __init__(self, units):
super(MyRNN, self).__init__()
# [b,300]=>[b,300,100]
self.embedding = layers.Embedding(total_words,
embedding_len,
input_length=max_review_len)
self.rnn = Sequential([
layers.SimpleRNN(units,
dropout=0.5,
return_sequences=True,
unroll=True),
layers.SimpleRNN(units, dropout=0.5)
])
# [b,64]=>[b,1]
self.outlayer = layers.Dense(1)
def recurrent_model(units=32, celltype='GRU', Tx=None, Trej=0, print_summary=False, learning_rate=.001, name='recurrent_model', loss='mse', metrics='mse', initializer = keras.initializers.GlorotNormal()):
'''
Recurrent neural network model.
'''
# specify recurrent cell type
if celltype=='GRU':
Rcell = layers.GRU(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
elif celltype=='LSTM':
Rcell = layers.LSTM(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
elif celltype=='RNN':
Rcell = layers.SimpleRNN(units=units, return_sequences=True, kernel_initializer=initializer, name='Xrec')
# Model architecture
X_input = layers.Input(shape=(Tx,1), name='X0')
X = Rcell(X_input)
X = layers.Dense(1, activation='linear', kernel_initializer=initializer, name='Xdense')(X)
Y = layers.Add(name='Y')([X,X_input])
# Create model
model = keras.Model(inputs=[X_input], outputs=Y, name=name)
if print_summary:
model.summary()
# Compile model
opt = tf.keras.optimizers.Adam(learning_rate,clipvalue=10)
model.compile(optimizer=opt, loss=loss, metrics=metrics)
return model
def build_rnn_model(embedding_dim,
vocabulary_size,
recurrent_type='SimpleRNN',
n_rnn_units=64,
n_rnn_layers=1,
bidirect=True):
tf.random.set_seed(123)
model = tf.keras.Sequential()
model.add(
tkl.Embedding(input_dim=vocabulary_size,
output_dim=embedding_dim,
name='embedding_layer'))
for i in range(n_rnn_layers):
return_sequences = (i < n_rnn_layers - 1)
if recurrent_type == 'SimpleRNN':
recurrent_layer = tkl.SimpleRNN(
units=n_rnn_units,
return_sequences=return_sequences,
name='simple_rnn_layer{}'.format(i))
elif recurrent_type == 'LSTM':
recurrent_layer = tkl.LSTM(units=n_rnn_units,
return_sequences=return_sequences,
name='lstm_layer{}'.format(i))
elif recurrent_type == 'GRU':
recurrent_layer = tkl.GRU(units=n_rnn_units,
return_seq=return_sequences,
name='gru_layer{}'.format(i))
if bidirect:
recurrent_layer = tkl.Bidirectional(recurrent_layer,
name='bidirect_' +
recurrent_layer.name)
model.add(recurrent_layer)
model.add(tkl.Dense(64, activation='relu'))
model.add(tkl.Dense(1, activation='sigmoid'))
return model
def __init__(self, units):
super().__init__()
self.embedding = layers.Embedding(TOTAL_WORD,
EMBEDDING_LEN,
input_length=MAX_REVIEW_LEN)
self.rnn = keras.Sequential([
layers.SimpleRNN(units, dropout=0.5, return_sequences=True),
layers.SimpleRNN(units, dropout=0.5)
])
self.outlayer = Sequential([
layers.Dense(32),
layers.Dropout(rate=0.5),
layers.ReLU(),
layers.Dense(1)
])
def fit(self,
x_train,
y_train,
modelPath,
epochs=10,
batch_size=100,
max_features=39,
maxlen=32,
embedding_value=4,
rnn_value=32):
self.model = keras.Sequential()
self.model.add(
layers.Embedding(max_features,
embedding_value,
input_length=maxlen))
self.model.add(layers.SimpleRNN(rnn_value))
self.model.add(layers.Dropout(0.4)) # change later
self.model.add(layers.Dense(1))
self.model.add(layers.Activation('sigmoid'))
self.model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
self.model.summary()
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
self.model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
self.model.save(modelPath)
def _build_model(self):
input = keras.Input(shape=(self.height, self.width, 1))
conv1 = layers.Conv2D(64, (3, 3), padding='same',
activation='relu')(input) #(batch,32,128,64)
pool1 = layers.MaxPool2D(pool_size=(2, 2), strides=2,
padding='same')(conv1)
conv2 = layers.Conv2D(128, (3, 3), padding='same',
activation='relu')(pool1) #(batch,16,64,128)
pool2 = layers.MaxPool2D(pool_size=(2, 2), strides=2,
padding='same')(conv2) #(batch,8,32,128)
trans = layers.Permute((2, 1, 3))(pool2) #(batch,32,8,128)
reshape = layers.Reshape((32, 1024))(trans) #(batch,32,1024)
RNN_in = layers.Dropout(0.5)(reshape)
# lstm1=layers.Bidirectional(layers.LSTM(31, return_sequences=True), backward_layer=layers.LSTM(31,return_sequences=True, go_backwards=True))(RNN_in) #(batch,31,512)
rnn = layers.SimpleRNN(256, return_sequences=True)(RNN_in)
# drop1=layers.Dropout(0.5)(lstm1)
# lstm2=layers.Bidirectional(layers.LSTM(31, return_sequences=True), backward_layer=layers.LSTM(31,return_sequences=True, go_backwards=True))(Drop1)
# lstm1=layers.LSTM(256, return_sequences=True, activation='sigmoid')(RNN_in)
# lstm2=layers.LSTM(256, return_sequences=True, activation='sigmoid')(bn4)
drop2 = layers.Dropout(0.5)(rnn)
logits = layers.Dense(self.num_of_classes,
activation='softmax',
kernel_constraint='UnitNorm',
use_bias=False)(drop2) #(batch,31,84)
# decoded, log_prob = tf.nn.ctc_beam_search_decoder(
# logits, [6,5])
# dense_decoded = tf.sparse.to_dense(
# decoded[0], default_value=-1, name="dense_decoded"
# )
self.model = keras.Model(inputs=input, outputs=logits)
self.model.compile(loss=self.loss.ctc_loss, optimizer=self.optimizer
) #,metrics=[self.loss.ctc_beam_decoder_loss])
def get_model_architecture(n_classes, rnn_layers, rnn_dim, fc_layers, fc_dim):
model = tf.keras.Sequential()
# The output of SimpleRNN will be a 2D tensor of shape (batch_size, 32)
# model.add(layers.SimpleRNN(32, input_shape=(config.n_mels, config.tisv_frame), return_sequences=True, activation='relu')) #, activation='relu'
for i in range(rnn_layers):
if i == rnn_layers - 1:
return_sequence = False
else:
return_sequence = True
# if lstm:
# model.add(
# layers.LSTM(rnn_dim, input_shape=(None, config.tisv_frame), return_sequences=return_sequence))
# else:
model.add(
layers.SimpleRNN(rnn_dim,
input_shape=(None, config.n_mels),
activation='relu',
return_sequences=return_sequence))
for i in range(fc_layers):
model.add(layers.Dense(fc_dim, activation='relu'))
# model.add(layers.Dense(len(np.unique(y_train)), activation='softmax'))
model.add(layers.Dense(n_classes, activation='softmax'))
return model
def get_model(bidirectional = False, seqModelType = "SimpleRNN", RNNunits = 32):
model = keras.Sequential()
model.add(layers.InputLayer(input_shape=(None,s)))
if seqModelType == "HMM":
seqLayer = HMMLayer(5, 15) # (10,15) is better than (5,11)
elif seqModelType == "LSTM":
seqLayer = layers.LSTM(RNNunits)
elif seqModelType == "GRU":
seqLayer = layers.GRU(RNNunits)
elif seqModelType == "SimpleRNN":
seqLayer = layers.SimpleRNN(RNNunits)
else:
sys.exit("unknown sequence model type " + seqModelType)
if bidirectional:
seqLayer = layers.Bidirectional(seqLayer)
model.add(seqLayer)
model.add(layers.Dense(1, activation='sigmoid'))
lr = 1e-3
#if seqModelType == "HMM":
# lr = 1e-2
print (f"lr={lr}")
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(), metrics = ["accuracy"])
return model
def rnn(n_outputs, window_size):
# 1-layer, basic model, doesn't work very well...
m = Sequential()
m.add(layers.InputLayer(input_shape=(window_size, 1)))
m.add(
layers.SimpleRNN(units=64,
activation='tanh',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.5,
recurrent_dropout=0.2,
return_sequences=False,
return_state=False,
go_backwards=False,
stateful=False,
unroll=False))
m.add(layers.BatchNormalization(name='batch_norm_1'))
# last layer
m.add(layers.Dense(n_outputs, activation='sigmoid', name='output'))
return m
def create_model():
model = keras.models.Sequential()
# 添加一个Masking层
model.add(layers.Masking(mask_value=0.0, input_shape=(2, 2)))
# 添加一个普通RNN层
rnn_layer = layers.SimpleRNN(50, return_sequences=False)
model.add(rnn_layer)
model.add(Dense(300, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(30, activation='relu'))
# 多个标签
# model.add(Dense(10, activation='sigmoid'))
# 单个标签
model.add(Dense(10, activation='sigmoid'))
adam = keras.optimizers.Adam(lr=0.05,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy', 'binary_accuracy'])
return model
def __init__(self, units):
super(MyRNN, self).__init__()
# 词向量编码 [b, 80] => [b, 80, 100]
self.embedding = layers.Embedding(total_words, embedding_len,
input_length=max_review_len)
# 构建RNN
self.rnn = keras.Sequential([
layers.SimpleRNN(units, dropout=0.5, return_sequences=True),
layers.SimpleRNN(units, dropout=0.5)
])
# 构建分类网络，用于将CELL的输出特征进行分类，2分类
# [b, 80, 100] => [b, 64] => [b, 1]
self.outlayer = Sequential([
layers.Dense(32),
layers.Dropout(rate=0.5),
layers.ReLU(),
layers.Dense(1)])
def __init__(self, unit):
super(MyRNN, self).__init__()
# embedding层，将字符串转换为数据 embedding_len就是embedding的长度，即一个单词用一个100维的向量表示
# [b, 80](b句话，每句话80个单词)->[b, 80, 100]（b句话，每句话80个单词，每个单词是100的维度）
self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
# 循环网络层.创建两层
self.rnn = keras.Sequential([
layers.SimpleRNN(unit, dropout=0.5,return_sequences=True, unroll=True),
layers.SimpleRNN(unit, dropout=0.5,unroll=True)
])
# [b, 80, 100] -> [b, 64]
# 全连接层,分类
self.outlayer = layers.Dense(1)
def construct(self, dropout_rate: float):
''' Construct the model.
- dropout_rate: Dropout rate.
'''
model_input = layers.Input(batch_shape=(None, N_STEPS, N_INPUT))
x = layers.SimpleRNN(N_NEURONS,
activation='elu',
return_sequences=True)(model_input)
output = layers.Dense(N_INPUT, activation='softmax', name='output')(x)
self.model = keras.Model(inputs=model_input, outputs=output)
