def net(self, x):
data_input = Input(shape=x.shape[1:])
f2,f3, f4, f5 = self.__call__(data_input)
net = Model(inputs=data_input, outputs=f5)
net.build(x.shape)
net.summary()
return net
def build_model():
state_emb_input = Input(shape=(1, ))
county_emb_input = Input(shape=(1, ))
data_input = Input(shape=(sequence_length, 3))
state_emb = Embedding(input_dim=n_state_labels,
output_dim=8,
input_length=1)(state_emb_input)
county_emb = Embedding(input_dim=n_county_labels,
output_dim=8,
input_length=1)(county_emb_input)
embedding = Concatenate()([state_emb, county_emb])
embedding = Flatten()(embedding)
embedding = Dense(sequence_length, activation='linear')(embedding)
embedding = Reshape((sequence_length, 1))(embedding)
concat = Concatenate()([data_input, embedding])
out = LSTM(128, dropout=0.2, return_sequences=True)(concat)
out = LSTM(256, dropout=0.2, return_sequences=False)(out)
out1 = Dense(1, activation='linear', name='Confirmed_Cases')(out)
out2 = Dense(1, activation='linear', name='Total_Deaths')(out)
model = Model([state_emb_input, county_emb_input, data_input],
[out1, out2])
model.build(input_shape=((1, ), (1, ), (sequence_length, 1)))
model.summary()
return model
def construct_bert(model_dir,
timesteps,
classes,
dense_dropout=0.5,
attention_dropout=0.3,
hidden_dropout=0.3,
adapter_size=8):
bert_ckpt_file = os.path.join(model_dir, "bert_model.ckpt")
bert_config_file = os.path.join(model_dir, "bert_config.json")
bert_params = bert.params_from_pretrained_ckpt(model_dir)
bert_model = bert.BertModelLayer.from_params(bert_params, name="bert")
input_ids = Input(shape=(timesteps, ), dtype='int32', name="input_ids_1")
token_type_ids = Input(shape=(timesteps, ),
dtype='int32',
name="token_type_ids_1")
dense = Dense(units=768, activation="tanh", name="dense")
output = bert_model([input_ids, token_type_ids
]) # output: [batch_size, max_seq_len, hidden_size]
print("bert shape", output.shape)
cls_out = Lambda(lambda seq: seq[:, 0:1, :])(output)
cls_out = Dropout(dense_dropout)(cls_out)
logits = dense(cls_out)
logits = Dropout(dense_dropout)(logits)
logits = Dense(units=classes, activation="softmax",
name="output_1")(logits)
model = Model(inputs=[input_ids, token_type_ids], outputs=logits)
model.build(input_shape=(None, timesteps))
# load the pre-trained model weights
load_stock_weights(bert_model, bert_ckpt_file)
return model
def build_model(self):
x = Input(shape=(self.input_shape, ))
outputs = self.regime(x, self.num_stages, self.hidden_size,
self.num_class, self.graph_model,
self.graph_param)
model = Model(inputs=[x], outputs=[outputs])
model.build(input_shape=[self.input_shape])
return model
def build_model(n_features):
print("--model--ly")
ly = MyLayer(10,2)
print("--model--Input")
input_layer = Input(shape=[10, 10], dtype='float32', name="input_ids")
print("--model--output")
output = ly(input_layer)
print("--model--new")
model = Model(inputs=input_layer, outputs=output)
print("--model--build")
model.build(input_shape=(None, n_features))
print("--model--compile")
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
return model
bert_layer = bert.BertModelLayer.from_params(bert_params, name="BERT")
bert_layer.apply_adapter_freeze()
print("Definisce la rete neurale transformer e ne stampa la struttura...")
# We arrange our layers by composing them as functions, with the input layer as inmost one
input_layer = Input(shape=(MAX_LENGTH, ), dtype='int32', name='input_ids')
output_layer = bert_layer(input_layer)
output_layer = GlobalAveragePooling1D()(output_layer)
output_layer = Dense(128, activation="relu")(output_layer)
output_layer = Dropout(0.1)(output_layer)
output_layer = Dense(1, activation="relu")(output_layer)
neural_network = Model(inputs=input_layer, outputs=output_layer)
neural_network.build(input_shape=(None, MAX_LENGTH))
neural_network.compile(loss="mse", optimizer=Adam(learning_rate=3e-5))
neural_network.summary()
try:
print("Se ho salvato in precedenza la rete la carico da file...")
neural_network.load_weights("pesi_rete")
except:
print("No! La alleno e poi la salvo...")
neural_network.fit(training_set,
training_scores,
batch_size=128,
shuffle=True,
epochs=4,
validation_data=(testing_set, testing_scores),
def CreateModel(self):
'''
定义CNN/LSTM/CTC模型，使用函数式模型
输入层：200维的特征值序列，一条语音数据的最大长度设为1600（大约16s）
隐藏层：卷积池化层，卷积核大小为3x3，池化窗口大小为2
隐藏层：全连接层
输出层：全连接层，神经元数量为self.MS_OUTPUT_SIZE，使用softmax作为激活函数，
CTC层：使用CTC的loss作为损失函数，实现连接性时序多输出
'''
input_data = Input(name='the_input',
shape=(self.AUDIO_LENGTH, self.AUDIO_FEATURE_LENGTH,
1))
layer_h1 = Conv2D(32, (3, 3),
use_bias=False,
activation='relu',
padding='same',
kernel_initializer='he_normal')(input_data) # 卷积层
layer_h1 = Dropout(0.05)(layer_h1)
layer_h2 = Conv2D(32, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h1) # 卷积层
layer_h3 = MaxPooling2D(pool_size=2, strides=None,
padding="valid")(layer_h2) # 池化层
#layer_h3 = Dropout(0.2)(layer_h2) # 随机中断部分神经网络连接，防止过拟合
layer_h3 = Dropout(0.05)(layer_h3)
layer_h4 = Conv2D(64, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h3) # 卷积层
layer_h4 = Dropout(0.1)(layer_h4)
layer_h5 = Conv2D(64, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h4) # 卷积层
layer_h6 = MaxPooling2D(pool_size=2, strides=None,
padding="valid")(layer_h5) # 池化层
layer_h6 = Dropout(0.1)(layer_h6)
layer_h7 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h6) # 卷积层
layer_h7 = Dropout(0.15)(layer_h7)
layer_h8 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h7) # 卷积层
layer_h9 = MaxPooling2D(pool_size=2, strides=None,
padding="valid")(layer_h8) # 池化层
layer_h9 = Dropout(0.15)(layer_h9)
layer_h10 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h9) # 卷积层
layer_h10 = Dropout(0.2)(layer_h10)
layer_h11 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h10) # 卷积层
layer_h12 = MaxPooling2D(pool_size=1, strides=None,
padding="valid")(layer_h11) # 池化层
layer_h12 = Dropout(0.2)(layer_h12)
layer_h13 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h12) # 卷积层
layer_h13 = Dropout(0.3)(layer_h13)
layer_h14 = Conv2D(128, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h13) # 卷积层
layer_h15 = MaxPooling2D(pool_size=1, strides=None,
padding="valid")(layer_h14) # 池化层
#test=Model(inputs = input_data, outputs = layer_h12)
#test.summary()
layer_h16 = Reshape((200, 3200))(layer_h15) #Reshape层
#layer_h5 = LSTM(256, activation='relu', use_bias=True, return_sequences=True)(layer_h4) # LSTM层
#layer_h6 = Dropout(0.2)(layer_h5) # 随机中断部分神经网络连接，防止过拟合
layer_h16 = Dropout(0.3)(layer_h16)
layer_h17 = Dense(128,
activation="relu",
use_bias=True,
kernel_initializer='he_normal')(layer_h16) # 全连接层
layer_h17 = Dropout(0.3)(layer_h17)
layer_h18 = Dense(self.MS_OUTPUT_SIZE,
use_bias=True,
kernel_initializer='he_normal')(layer_h17) # 全连接层
y_pred = Activation('softmax', name='Activation0')(layer_h18)
model_data = Model(inputs=input_data, outputs=y_pred)
#model_data.summary()
labels = Input(name='the_labels',
shape=[self.label_max_string_length],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# tensorflow.keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
#layer_out = Lambda(ctc_lambda_func,output_shape=(self.MS_OUTPUT_SIZE, ), name='ctc')([y_pred, labels, input_length, label_length])#(layer_h6) # CTC
loss_out = Lambda(self.ctc_lambda_func, output_shape=(1, ),
name='ctc')(
[y_pred, labels, input_length, label_length])
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
model.summary()
# clipnorm seems to speeds up convergence
#sgd = SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
#ada_d = Adadelta(lr = 0.01, rho = 0.95, epsilon = 1e-06)
opt = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
decay=0.0,
epsilon=10e-8)
#model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
model.build((self.AUDIO_LENGTH, self.AUDIO_FEATURE_LENGTH, 1))
model = ParallelModel(model, NUM_GPU)
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=opt)
# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])
#print('[*提示] 创建模型成功，模型编译成功')
print('[*Info] Create Model Successful, Compiles Model Successful. ')
return model, model_data
return model
# -
dim_hidden_layres = [2**i for i in range(4, 8)]
# +
df_accuracy = pd.DataFrame()
for hid_dim_0 in dim_hidden_layres:
for hid_dim_1 in dim_hidden_layres:
print('========', 'hid_dim_0:', hid_dim_0, '; hid_dim_1:', hid_dim_1,
'========')
model = CNNModel(hid_dim_0=hid_dim_0, hid_dim_1=hid_dim_1)
model = model.build()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
callbacks = [
EarlyStopping(patience=3),
ModelCheckpoint(filepath=os.path.join(
'models', 'CNN', 'model_{}_{}.h5'.format(hid_dim_0,
hid_dim_1)),
save_best_only=True),
]
n_param = model.count_params()
model.fit(x=x_train,
y=y_train,
batch_size=128,
epochs=100,
kernel_regularizer=l2(0.01),
bias_regularizer=l2(0.01))(x) # dense layer 3
x = Dropout(0.5)(x)
preds = Dense(num_classes,
activation='softmax')(x) # final layer with softmax activation
model = Model(inputs=base_model.input, outputs=preds)
for layer in model.layers[:20]:
layer.trainable = False
for layer in model.layers[20:]:
layer.trainable = True
model.load_weights(
'/home/ivo/Projects/auto_image_rotator/auto_image_flip/app/static/saved_model/image_rotate_weights.h5'
)
optimizer = tf.optimizers.Adam(1e-5)
model.build([None, 224, 224, 3])
model.summary()
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
# Convert model to TFLite
TFLITE_MODEL = "models/export/image_flip.tflite"
TFLITE_QUANT_MODEL = "models/export/image_flip_quant.tflite"
# Get the concrete function from the Keras model.
run_model = tf.function(lambda x: model(x))
def build_bilstm(start_pointer,
end_pointer,
emb_dim,
max_passage_length=None,
max_query_length=None,
num_highway_layer=2,
num_decoder=1,
encoder_dropout=0,
decoder_dropout=0):
passage_input = Input(shape=(max_passage_length, emb_dim),
dtype='float32',
name='passage_input')
query_input = Input(shape=(max_query_length, emb_dim),
dtype='float32',
name='query_input')
passage_embedding = passage_input
query_embedding = query_input
for i in range(num_highway_layer):
highway_layer = Highway(name='highway_{}'.format(i))
query_layer = TimeDistributed(highway_layer,
name=highway_layer.name + '_qtd')
query_embedding = query_layer(query_embedding)
passage_layer = TimeDistributed(highway_layer,
name=highway_layer.name + '_ptd')
passage_embedding = passage_layer(passage_embedding)
encoder_layer_p = Bidirectional(LSTM(emb_dim,
recurrent_dropout=encoder_dropout,
return_sequences=True),
name='bidirectional_encoder_p')
encoder_layer_q = Bidirectional(LSTM(emb_dim,
recurrent_dropout=encoder_dropout,
return_sequences=True),
name='bidirectional_encoder_q')
passage_encoded = encoder_layer_p(passage_embedding)
query_encoded = encoder_layer_q(query_embedding)
passage_outs = []
query_outs = []
num_lstm_layers = 3
for i in range(num_lstm_layers):
deca_encoder_p = Bidirectional(LSTM(emb_dim, return_sequences=True),
name='deca_encoder_p_{}'.format(i))
deca_encoder_q = Bidirectional(LSTM(emb_dim, return_sequences=True),
name='deca_encoder_q_{}'.format(i))
passage_deca = deca_encoder_p(passage_encoded)
query_deca = deca_encoder_q(query_encoded)
passage_outs.append(passage_deca)
query_outs.append(query_deca)
passage_c = tf.concat(passage_outs, 2)
query_c = tf.concat(query_outs, 2)
passage_connecters = []
query_connecters = []
for i in range(num_lstm_layers):
for j in range(num_lstm_layers):
bac_layer = BAC(name='bac_layer_p_{}_q_{}'.format(i, j))
connecter_p, connecter_q = bac_layer(
tf.concat([passage_outs[i], query_outs[j]], 1))
passage_connecters.append(connecter_p)
query_connecters.append(connecter_q)
connecter_pc = tf.concat(passage_connecters, 2)
connecter_qc = tf.concat(query_connecters, 2)
passage_decaout = tf.concat([passage_c, connecter_pc], 2)
query_decaout = tf.concat([query_c, connecter_qc], 2)
print(passage_decaout.shape, query_decaout.shape)
gated_attention_layer = Gated_attention_with_self()
gated_atten_op, gated_self_atten_op = gated_attention_layer(
tf.concat([passage_decaout, query_decaout], 1))
atten_outs = [gated_atten_op, gated_self_atten_op]
conn_list = []
print(atten_outs[0].shape, query_outs[0].shape)
for i in range(2):
for j in range(num_lstm_layers):
one_sided_bac = BAC(name='one_sided_bac_u{}_q{}'.format(i, j))
connecter_1, connecter_2 = one_sided_bac(
tf.concat([atten_outs[i], query_outs[j]], 1))
conn_list.append(connecter_1)
# conn_list.append(connecter_2) #TODO: Check Upper BAC in code.
for k in range(len(conn_list)):
print(conn_list[k].shape)
conn_c = tf.concat(conn_list, 2)
m_mat = tf.concat([gated_self_atten_op, conn_c], 2)
start_projection = Bidirectional(
LSTM(emb_dim, name='bilstm_span_start', return_sequences=True))
end_projection = Bidirectional(
LSTM(emb_dim, name='bilstm_span_end', return_sequences=True))
softamx_start = Dense(1,
activation=softmax,
use_bias=False,
name='start_softmax')
softamx_end = Dense(1,
activation=softmax,
use_bias=False,
name='end_softmax')
span_start = start_projection(m_mat)
span_end = end_projection(span_start)
start_pred = softamx_start(span_start)
end_pred = softamx_end(span_end)
# start_pointer = tf.zeros((32, 200, 1))
# end_pointer = tf.zeros((32, 200, 1))
# loss_start = tf.math.log(tf.reduce_sum(tf.multiply(start_pred, start_pointer)))
# loss_end = tf.math.log(tf.reduce_sum(tf.multiply(end_pred, end_pointer)))
loss_start = tf.math.log(
tf.reduce_sum(tf.multiply(start_pred, start_pred), axis=1))
loss_end = tf.math.log(
tf.reduce_sum(tf.multiply(end_pred, end_pred), axis=1))
cost = tf.reduce_mean(tf.math.add(loss_start, loss_end), axis=0)
print(loss_start.shape, loss_end.shape, cost.shape)
model = Model(inputs=[passage_input, query_input],
outputs=[start_pred, end_pred])
model.build(input_shape=(max_passage_length, emb_dim))
model.summary()