testY[testY == -1] = 0
testX = np.reshape(testX, [-1, 10, 2056])
testX_pos = testX[:, :, 0:8]
testX_img = testX[:, :, 8:]
testY = np.reshape(to_categorical(np.reshape(testY, (-1, )), num_classes=2),
(-1, 10, 2))
print(testY.shape, testX_img.shape)
# person_lstm = load_model('./split4/models/person-lstm3-split4.h5')
# keras.backend.set_value(person_lstm.optimizer.lr, 0.0001*2)
pos_in = Input(shape=(10, 8))
img_in = Input(shape=(10, 2048))
pos_masked = Masking()(pos_in)
img_masked = Masking()(img_in)
lstm_pos = LSTM(512,
activation='sigmoid',
recurrent_activation='tanh',
return_sequences=True)(pos_masked)
lstm_img = LSTM(256,
activation='sigmoid',
recurrent_activation='tanh',
return_sequences=True)(img_masked)
drop_img = Dropout(0.95)(lstm_img)
drop_pos = Dropout(0.01)(lstm_pos)
fc_pos = (Dense(128, activation='tanh')(drop_pos))
def GRU_only_Feature_model():
input_featrue_noraml = Input((max_par_num, normal_feature_num),
name='feature_normal')
input_bianhao = Input((max_par_num, ), name='bianhao')
emb_bianhao = Embedding(max_bianhao,
max_bianhao,
mask_zero=True,
weights=[weight_bianhao])(input_bianhao)
input_wordduixiang = Input((max_par_num, ), name='wordduixiang')
emb_wordduixiang = Embedding(max_wordduixiang,
max_wordduixiang,
mask_zero=True,
weights=[weight_wordduixiang
])(input_wordduixiang)
input_bianhaoweizhi = Input((max_par_num, ), name='bianhaoweizhi')
emb_bianhaoweizhi = Embedding(max_bianhaoweizhi,
max_bianhaoweizhi,
mask_zero=True,
weights=[weight_bianhaoweizhi
])(input_bianhaoweizhi)
input_keyword = Input((max_par_num, ), name='keyword')
emb_keyword = Embedding(max_keyword,
max_keyword,
mask_zero=True,
weights=[weight_keyword])(input_keyword)
input_duiqifangshi = Input((max_par_num, ), name='duiqifangshi')
emb_duiqifangshi = Embedding(max_duiqifangshi,
max_duiqifangshi,
mask_zero=True,
weights=[weight_duiqifangshi
])(input_duiqifangshi)
input_dagangjibie = Input((max_par_num, ), name='dagangjibie')
emb_dagangjibie = Embedding(max_dagangjibie,
max_dagangjibie,
mask_zero=True,
weights=[weight_dagangjibie
])(input_dagangjibie)
fe = concatenate([
input_featrue_noraml, emb_bianhaoweizhi, emb_dagangjibie,
emb_duiqifangshi, emb_keyword, emb_wordduixiang, emb_bianhao
])
# input_featrue_noraml = Input((max_par_num,normal_feature_num),name='feature_normal')
# input_bianhao = Input((max_par_num,),name='bianhao')
# emb_bianhao = Embedding(max_bianhao,max_bianhao,mask_zero=True)(input_bianhao)
# input_wordduixiang = Input((max_par_num,),name='wordduixiang')
# emb_wordduixiang = Embedding(max_wordduixiang,max_wordduixiang,mask_zero=True)(input_wordduixiang)
# input_bianhaoweizhi = Input((max_par_num,),name='bianhaoweizhi')
# emb_bianhaoweizhi = Embedding(max_bianhaoweizhi,max_bianhaoweizhi,mask_zero=True)(input_bianhaoweizhi)
# input_keyword = Input((max_par_num,),name='keyword')
# emb_keyword = Embedding(max_keyword,max_keyword,mask_zero=True)(input_keyword)
# input_duiqifangshi = Input((max_par_num,),name='duiqifangshi')
# emb_duiqifangshi = Embedding(max_duiqifangshi,max_duiqifangshi,mask_zero=True)(input_duiqifangshi)
# input_dagangjibie = Input((max_par_num,),name='dagangjibie')
# emb_dagangjibie = Embedding(max_dagangjibie,max_dagangjibie,mask_zero=True)(input_dagangjibie)
# fe = concatenate([emb_bianhaoweizhi,emb_dagangjibie,emb_duiqifangshi,emb_keyword,emb_wordduixiang,emb_bianhao])
print(fe.shape)
fe = Masking(mask_value=0)(fe)
x = Bidirectional(GRU(32, return_sequences=True, dropout=0.2))(fe)
# x = Bidirectional(GRU(128, return_sequences=True,dropout=0.2))(x)
x = TimeDistributed(Dense(num_class))(x)
crf = CRF(num_class, sparse_target=False, learn_mode='join')
x = crf(x)
model = Model(inputs=[
input_featrue_noraml, input_wordduixiang, input_bianhaoweizhi,
input_keyword, input_duiqifangshi, input_dagangjibie, input_bianhao
],
output=x)
# myadam = adam(lr=0.001,decay=1e-6)
model.compile(
loss=crf.loss_function,
# loss='categorical_crossentropy',
optimizer='adam',
metrics=[crf.accuracy])
print(model.summary())
return model
def build_encoder_decoder_inference(self,
model,
sentence_encoder,
include_sentence_encoder=True,
attention=False):
latent_dim = 0
initial_encoder_states = []
initial_input = []
encoder_inputs = Input(shape=model.get_layer(
"encoder_input_layer").get_config()['batch_input_shape'][1:])
print("encoder inputs shape: ", encoder_inputs.shape)
mask_layer = Masking(mask_value=0, name="mask_layer")
mask_output = mask_layer(encoder_inputs)
encoder_lstm_prefix = "encoder_layer_"
num_encoder = self.get_number_of_layers(model, encoder_lstm_prefix)
print("num: ", num_encoder)
for i in range(num_encoder):
encoder = model.get_layer(encoder_lstm_prefix + str(i))
weights = encoder.get_weights()
config = encoder.get_config()
config['dropout'] = 0.0
config['recurrent_dropout'] = 0.0
encoder = layers.deserialize({
'class_name': encoder.__class__.__name__,
'config': config
})
if i == 0:
encoder_outputs = encoder(mask_output)
encoder.set_weights(weights)
latent_dim = encoder.get_config()['units']
else:
encoder_outputs = encoder(encoder_outputs[0])
encoder.set_weights(weights)
encoder_states = encoder_outputs[1:]
if include_sentence_encoder:
encoder_sentence_inputs = Input(shape=(22, ))
initial_input = [encoder_inputs, encoder_sentence_inputs]
sentence_encoder_embedding_layer = model.get_layer(
'sentence_embedding_layer')
sentence_embedding_outputs = sentence_encoder_embedding_layer(
encoder_sentence_inputs)
if attention:
sentence_encoder_outputs, initial_encoder_states, new_latent_dim = sentence_encoder.get_last_layer_inference(
model,
encoder_states,
sentence_embedding_outputs,
attention=attention)
#this is just for now, because the our model only accepts only the hidden state from the image encoder
initial_encoder_states = encoder_states[0]
else:
initial_encoder_states, new_latent_dim = sentence_encoder.get_last_layer_inference(
model, encoder_states, sentence_embedding_outputs)
else:
initial_input = encoder_inputs
initial_encoder_states = encoder_states
new_latent_dim = latent_dim
if attention:
encoder_model = Model(
initial_input,
[sentence_encoder_outputs, initial_encoder_states])
else:
encoder_model = Model(initial_input, initial_encoder_states)
decoder_inputs = Input(shape=(None, ))
embedding_layer = model.get_layer("decoder_embedding_layer")
embedding_outputs = embedding_layer(decoder_inputs)
decoder_prefix = "decoder_layer_"
num_decoder = self.get_number_of_layers(model, decoder_prefix)
if len(encoder_states) == 1:
# decoder_states_inputs = [decoder_state_input_h1, decoder_state_input_h2]
decoder_states_inputs = []
for i in range(num_decoder):
decoder_states_inputs.append(Input(shape=(new_latent_dim, )))
else:
# decoder_states_inputs = [decoder_state_input_h1, decoder_state_input_c]
decoder_states_inputs = []
for i in range(num_decoder):
decoder_states_inputs.append(Input(shape=(new_latent_dim, )))
decoder_states_inputs.append(Input(shape=(new_latent_dim, )))
decoder_states_inputs.append(
Input(shape=(
sentence_encoder_outputs.shape[1],
sentence_encoder_outputs.shape[2],
)))
decoder_states = []
decoder_outputs = embedding_outputs
for i in range(num_decoder):
decoder = model.get_layer(decoder_prefix + str(i))
weights = decoder.get_weights()
config = decoder.get_config()
config['dropout'] = 0.0
config['recurrent_dropout'] = 0.0
if attention:
if i == num_decoder - 1:
decoder = layers.deserialize(
{
'class_name': decoder.__class__.__name__,
'config': config
},
custom_objects={'AttentionGRU': AttentionGRU})
decoder_outputs = decoder(
decoder_outputs,
initial_state=decoder_states_inputs[i],
constants=decoder_states_inputs[-1])
else:
decoder = layers.deserialize({
'class_name':
decoder.__class__.__name__,
'config':
config
})
decoder_outputs = decoder(
decoder_outputs,
initial_state=decoder_states_inputs[i])
else:
decoder = layers.deserialize({
'class_name': decoder.__class__.__name__,
'config': config
})
decoder_outputs = decoder(
decoder_outputs, initial_state=decoder_states_inputs[i])
decoder.set_weights(weights)
decoder_states = decoder_states + list(decoder_outputs[1:])
decoder_outputs = decoder_outputs[0]
decoder_dense = model.get_layer("dense_layer")
decoder_outputs = decoder_dense(decoder_outputs)
decoder_model = Model([decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
# return im_model, sent_model
return encoder_model, decoder_model
def run(times=1):
try:
x, y = load_xy()
except:
x, y = build_xy()
save_xy(x, y)
num_timesteps = x.shape[1]
num_classes = y.shape[1]
num_samples = len(x)
num_train = int(math.floor(num_samples * 0.7))
num_test = num_samples - num_train
seeds = random.sample(range(100000), times)
print('num samples: ', num_samples)
print('num timesteps per sample: ', num_timesteps)
print('num features per timestep: ', num_features)
print('num output classes: ', num_classes)
print('num train samples: ', num_train)
print('num test samples: ', num_test)
print('seeds: ', seeds)
accuracy_results = []
for run_iteration in range(0, times):
seeded_shuffle(x, seeds[run_iteration])
seeded_shuffle(y, seeds[run_iteration])
x_train = x[:num_train]
y_train = y[:num_train]
x_test = x[num_train:]
y_test = y[num_train:]
model = Sequential()
model.add(
Conv1D(filters=64,
kernel_size=3,
activation='relu',
input_shape=(num_timesteps, num_features)))
model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))
# Dropout consists in randomly setting a fraction rate of input units
# to 0 at each update during training time, which helps prevent overfitting.
model.add(Dropout(0.5))
# Downsamples the input. Calculate the maximum value for each patch of the feature map.
model.add(MaxPooling1D(pool_size=2))
# Flattens the input. Does not affect the batch size.
model.add(Flatten())
# If all features for a given sample timestep are equal to mask_value,
# then the sample timestep will be masked (skipped) in all downstream
# layers (as long as they support masking).
model.add(Masking(mask_value=0.0))
# Just your regular densely-connected NN layer.
model.add(Dense(100, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=13, batch_size=32)
_, accuracy = model.evaluate(x_test, y_test)
print('accuracy: ', accuracy)
accuracy_results.append(accuracy)
if run_iteration == times - 1:
print('\n')
print('results: ', accuracy_results)
def get_text_model(self):
# Modality specific hyperparameters
self.epochs = 5
self.batch_size = 50
# Modality specific parameters
self.embedding_dim = self.data.W.shape[1]
# For text model
self.vocabulary_size = self.data.W.shape[0]
self.filter_sizes = [3, 4, 5]
self.num_filters = 512
print("Creating Model...")
sentence_length = self.train_x.shape[2]
# Initializing sentence representation layers
embedding = Embedding(input_dim=self.vocabulary_size,
output_dim=self.embedding_dim,
weights=[self.data.W],
input_length=sentence_length,
trainable=False)
conv_0 = Conv2D(self.num_filters,
kernel_size=(self.filter_sizes[0], self.embedding_dim),
padding='valid',
kernel_initializer='normal',
activation='relu')
conv_1 = Conv2D(self.num_filters,
kernel_size=(self.filter_sizes[1], self.embedding_dim),
padding='valid',
kernel_initializer='normal',
activation='relu')
conv_2 = Conv2D(self.num_filters,
kernel_size=(self.filter_sizes[2], self.embedding_dim),
padding='valid',
kernel_initializer='normal',
activation='relu')
maxpool_0 = MaxPool2D(pool_size=(sentence_length -
self.filter_sizes[0] + 1, 1),
strides=(1, 1),
padding='valid')
maxpool_1 = MaxPool2D(pool_size=(sentence_length -
self.filter_sizes[1] + 1, 1),
strides=(1, 1),
padding='valid')
maxpool_2 = MaxPool2D(pool_size=(sentence_length -
self.filter_sizes[2] + 1, 1),
strides=(1, 1),
padding='valid')
dense_func = Dense(100, activation='tanh', name="dense")
dense_final = Dense(units=self.classes, activation='softmax')
reshape_func = Reshape((sentence_length, self.embedding_dim, 1))
def slicer(x, index):
return x[:, K.constant(index, dtype='int32'), :]
def slicer_output_shape(input_shape):
shape = list(input_shape)
assert len(shape) == 3 # batch, seq_len, sent_len
new_shape = (shape[0], shape[2])
return new_shape
def reshaper(x):
return K.expand_dims(x, axis=3)
def flattener(x):
x = K.reshape(x, [-1, x.shape[1] * x.shape[3]])
return x
def flattener_output_shape(input_shape):
shape = list(input_shape)
new_shape = (shape[0], 3 * shape[3])
return new_shape
inputs = Input(shape=(self.sequence_length, sentence_length),
dtype='int32')
cnn_output = []
for ind in range(self.sequence_length):
local_input = Lambda(slicer,
output_shape=slicer_output_shape,
arguments={"index": ind
})(inputs) # Batch, word_indices
# cnn-sent
emb_output = embedding(local_input)
reshape = Lambda(reshaper)(emb_output)
concatenated_tensor = Concatenate(axis=1)([
maxpool_0(conv_0(reshape)),
maxpool_1(conv_1(reshape)),
maxpool_2(conv_2(reshape))
])
flatten = Lambda(
flattener,
output_shape=flattener_output_shape,
)(concatenated_tensor)
dense_output = dense_func(flatten)
dropout = Dropout(0.5)(dense_output)
cnn_output.append(dropout)
def stack(x):
return K.stack(x, axis=1)
cnn_outputs = Lambda(stack)(cnn_output)
masked = Masking(mask_value=0)(cnn_outputs)
lstm = Bidirectional(
LSTM(300, activation='relu', return_sequences=True,
dropout=0.3))(masked)
lstm = Bidirectional(LSTM(300,
activation='relu',
return_sequences=True,
dropout=0.3),
name="utter")(lstm)
output = TimeDistributed(Dense(self.classes,
activation='softmax'))(lstm)
model = Model(inputs, output)
return model
def hello(request):
id = request.GET.get('id', '0')
print(str(id))
import numpy as np
import keras
from keras.models import Sequential
from keras.models import Model
from keras.layers import Masking, Embedding, Bidirectional, LSTM, Dense, Input, TimeDistributed, Activation
from keras.preprocessing import sequence
from keras_contrib.layers import CRF
from keras_contrib.losses import crf_loss
from keras_contrib.metrics import crf_viterbi_accuracy
from keras import backend as K
import argparse
import pymysql
import codecs
import json
char_vocab_path = "E:/study/kg/some_example/ner/ner_web_v1/static/data/char_vocabs.txt" # 字典文件
train_data_path = "./static/data/train_data.txt" # 训练数据
test_data_path = "./static/data/test_data.txt" # 测试数据
# 设置参数
# ap = argparse.ArgumentParser()
# ap.add_argument("-i", "--url", required=True,help="path to the text url")
# args = vars(ap.parse_args())
# TextUrl = args['url']
# with open(TextUrl,"r",encoding='utf-8') as f:
# str = f.read()
# print(str)
# s=[]
# s.append(str)
'''数据库连接'''
# 根据流程
# 1.我们先建立数据库的连接信息
# host = "127.0.0.1"
# user = "******"
# password = "******"
# port = 3306
# mysql = pymysql.connect(host=host, user=user, password=password, port=port)
# 2.新建个查询页面
# cursor = mysql.cursor()
# 3编写sql
# sql = 'SELECT CONTENT_TXT FROM literaturesystem.DOCUMENTS WHERE UUID = ' + id
# 4.执行sql
# cursor.execute(sql)
# 5.查看结果
# result = cursor.fetchone() #用于返回单条数据
# results = cursor.fetchall() # 用于返回多条数据
if re.findall("<br/>", str(id)):
TM_TEXT = str(id).split("<br/>")
elif re.findall("\r\n", str(id)):
TM_TEXT = str(id).split("\r\n")
else:
TM_TEXT = []
TM_TEXT.append(str(id))
print(TM_TEXT)
# print(results[0][0])
# print(TM_TEXT)
# print(TM_TEXT)
s = []
for data in TM_TEXT:
s.append(data)
# 6.关闭查询
# cursor.close()
# 关闭数据库
# mysql.close()
special_words = ['<PAD>', '<UNK>'] # 特殊词表示
# "BIO"标记的标签
label2idx = {
"O": 0,
"B-Dtype": 1,
"I-Dtype": 2,
"B-Dmaterial": 3,
"I-Dmaterial": 4,
"B-DmaterialsStrength": 5,
"I-DmaterialsStrength": 6,
"B-Dshape": 7,
"I-Dshape": 8,
"B-Dcrh": 9,
"I-Dcrh": 10,
"B-Ddiameter": 11,
"I-Ddiameter": 12,
"B-Dlength": 13,
"I-Dlength": 14,
"B-Dweight": 15,
"I-Dweight": 16,
"B-Zspeed": 17,
"I-Zspeed": 18,
"B-Zangle": 19,
"I-Zangle": 20,
"B-Btype": 21,
"I-Btype": 22,
"B-Bthickness": 23,
"I-Bthickness": 24,
"B-Bstrength": 25,
"I-Bstrength": 26,
"B-Bdensity": 27,
"I-Bdensity": 28,
"B-Bratio": 29,
"I-Bratio": 30,
"B-Xdepth": 31,
"I-Xdepth": 32,
"B-Xpenetrate": 33,
"I-Xpenetrate": 34,
"B-F": 33,
"I-F": 34
}
# 索引和BIO标签对应
idx2label = {idx: label for label, idx in label2idx.items()}
# 读取字符词典文件
with open(char_vocab_path, "r", encoding="utf8") as fo:
char_vocabs = [line.strip() for line in fo]
char_vocabs = special_words + char_vocabs
# 字符和索引编号对应
idx2vocab = {idx: char for idx, char in enumerate(char_vocabs)}
vocab2idx = {char: idx for idx, char in idx2vocab.items()}
EPOCHS = 1
BATCH_SIZE = 50
EMBED_DIM = 1
HIDDEN_SIZE = 1
MAX_LEN = 10
VOCAB_SIZE = len(vocab2idx)
CLASS_NUMS = len(label2idx)
## BiLSTM+CRF模型构建
inputs = Input(shape=(MAX_LEN, ), dtype='int32')
x = Masking(mask_value=0)(inputs)
x = Embedding(VOCAB_SIZE, EMBED_DIM, mask_zero=True)(x)
x = Bidirectional(LSTM(HIDDEN_SIZE, return_sequences=True))(x)
x = TimeDistributed(Dense(CLASS_NUMS))(x)
outputs = CRF(CLASS_NUMS)(x)
model = Model(inputs=inputs, outputs=outputs)
model.summary()
def f1(y_true, y_pred):
def recall(y_true, y_pred):
"""Recall metric.
Only computes a batch-wise average of recall.
Computes the recall, a metric for multi-label classification of
how many relevant items are selected.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def precision(y_true, y_pred):
"""Precision metric.
Only computes a batch-wise average of precision.
Computes the precision, a metric for multi-label classification of
how many selected items are relevant.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
precision = precision(y_true, y_pred)
recall = recall(y_true, y_pred)
return 2 * ((precision * recall) / (precision + recall + K.epsilon()))
'''用来预测'''
from keras.models import load_model
import numpy as np
custom_ob = {
'CRF': CRF,
"crf_loss": crf_loss,
"crf_viterbi_accuracy": crf_viterbi_accuracy,
"f1": f1
}
keras.backend.clear_session()
model = load_model(
'E:/study/kg/some_example/ner/01bilstm-ner/model/ch_ner_model4.h5',
custom_objects=custom_ob)
maxlen = 500
# sentence = "中华人民共和国国务院总理周恩来在外交部长陈毅的陪同下，连续访问了埃塞俄比亚等非洲10国以及阿尔巴尼亚。"
# sentence = "弹体为新型缩比钻地弹，弹体材料为DT300高强度合金钢，" \
# "抗拉强度为1810MPa，内部装填物为高分子惰性材料，弹体直径25mm，" \
# "长径比为6，弹壳壁厚与弹径比为0.15，弹体质量约340g，如图2所示。"
# sentence = "验采用Φ30mm口径射弹垂直侵彻，弹长138mm，弹径30mm，弹重0.5kg，选择适当的药量，可以获得需要的弹速，试验中射弹的着靶速度控制在276~456m/s范围内。"
# s = ["弹体为新型缩比钻地弹，弹体材料为DT300高强度合金钢,抗拉强度为1810MPa，内部装填物为高分子惰性材料，弹体直径25mm。长径比为6，弹壳壁厚与弹径比为0.15，弹体质量约340g，如图2所示。混凝土靶直径为100m，长径比为6。在本次的实验中混凝土靶标的长度为200cm.",
# "试验弹体直径为10mm，质量约为50g，长径比为6，弹壳壁厚与弹径比为0.15，弹体质量约340g，如图2所示。混凝土靶直径为100m，长径比为6。在本次的实验中混凝土靶标的长度为200cm.",
# "设计了一款Φ1000mm的试验弹丸，长径比为6，弹壳壁厚与弹径比为0.15，弹体质量约340g，如图2所示，混凝土靶直径为100m，长径比为6。在本次的实验中混凝土靶标的长度为200cm.",
# "混凝土靶直径为100m，长径比为6。在本次的实验中混凝土靶标的质量为100kg."
# ]
TextAll = []
for i in s:
sent_chars = list(i)
sent2id = [
vocab2idx[word] if word in vocab2idx else vocab2idx['<UNK>']
for word in sent_chars
]
sent2id_new = np.array([[0] * (maxlen - len(sent2id)) +
sent2id[:maxlen]])
y_pred = model.predict(sent2id_new)
y_label = np.argmax(y_pred, axis=2)
y_label = y_label.reshape(1, -1)[0]
y_ner = [idx2label[i] for i in y_label][-len(sent_chars):]
# print(idx2label)
# print(sent_chars)
# print(sent2id)
# print(y_ner)
# 对预测结果进行命名实体解析和提取
def get_valid_nertag(input_data, result_tags):
result_words = []
start, end = 0, 1 # 实体开始结束位置标识
tag_label = "O" # 实体类型标识
for i, tag in enumerate(result_tags):
if tag.startswith("B"):
if tag_label != "O": # 当前实体tag之前有其他实体
result_words.append(
(input_data[start:end], tag_label)) # 获取实体
tag_label = tag.split("-")[1] # 获取当前实体类型
start, end = i, i + 1 # 开始和结束位置变更
elif tag.startswith("I"):
temp_label = tag.split("-")[1]
if temp_label == tag_label: # 当前实体tag是之前实体的一部分
end += 1 # 结束位置end扩展
elif tag == "O":
if tag_label != "O": # 当前位置非实体 但是之前有实体
result_words.append(
(input_data[start:end], tag_label)) # 获取实体
tag_label = "O" # 实体类型置"O"
start, end = i, i + 1 # 开始和结束位置变更
if tag_label != "O": # 最后结尾还有实体
result_words.append(
(input_data[start:end], tag_label)) # 获取结尾的实体
return result_words
Ddiameter = []
result_words = get_valid_nertag(sent_chars, y_ner)
Dtype = []
Dmaterial = []
DmaterialsStrength = []
Dshape = []
Dcrh = []
Ddiameter = []
Dlength = []
Dweight = []
Zspeed = []
Zangle = []
Btype = []
Bthickness = []
Bstrength = []
Bdensity = []
Bratio = []
Xdepth = []
Xpenetrate = []
F = []
for (word, tag) in result_words:
# 一条数据
if tag == "Dtype":
# print("Dtype：", "".join(word))
Dtype.append("".join(word))
if tag == "Dmaterial":
# print("Dmaterial：","".join(word))
Dmaterial.append("".join(word))
if tag == "DmaterialsStrength":
# print("DmaterialsStrength：","".join(word))
DmaterialsStrength.append("".join(word))
if tag == "Dshape":
# print("Dshape：","".join(word))
Dshape.append("".join(word))
if tag == "Dcrh":
# print("Dcrh：","".join(word))
Dcrh.append("".join(word))
if tag == "Ddiameter":
# print("Ddiameter：","".join(word))
Ddiameter.append(word)
if tag == "Dlength":
# print("Dlength：","".join(word))
Dlength.append("".join(word))
if tag == "Dweight":
# print("Dweight：","".join(word))
Dweight.append("".join(word))
if tag == "Zspeed":
# print("Zspeed：","".join(word))
Zspeed.append("".join(word))
if tag == "Zangle":
# print("Zangle：","".join(word))
Zangle.append("".join(word))
if tag == "Btype":
# print("Btype：","".join(word))
Btype.append("".join(word))
if tag == "Bthickness":
# print("Bthickness：","".join(word))
Bthickness.append("".join(word))
if tag == "Bstrength":
# print("Bstrength：","".join(word))
Bstrength.append("".join(word))
if tag == "Bdensity":
# print("Bdensity：","".join(word))
Bdensity.append("".join(word))
if tag == "Bratio":
# print("Bratio：","".join(word))
Bratio.append("".join(word))
if tag == "Xdepth":
# print("Xdepth：","".join(word))
Xdepth.append("".join(word))
if tag == "Xpenetrate":
# print("Xpenetrate：","".join(word))
Xpenetrate.append("".join(word))
if tag == "F":
# print("F：","".join(word))
F.append("".join(word))
TextLine = {
'弹体类型': Dtype,
'弹体材料': Dmaterial,
'弹体材料强度': DmaterialsStrength,
'弹头形状': Dshape,
'CRH ': Dcrh,
'弹体直径': Ddiameter,
'弹体长度': Dlength,
'弹体质量': Dweight,
'着靶速度': Zspeed,
'命中角': Zangle,
'靶标材料种类': Btype,
'靶标厚度': Bthickness,
'靶标抗压强度': Bstrength,
'靶标材料密度': Bdensity,
'靶标配筋率': Bratio,
'侵彻深度': Xdepth,
'贯穿': Xpenetrate,
'发射炮类型': F
}
TextAll.append(TextLine)
# length=[len(Dtype),len(Dmaterial),len(DmaterialsStrength),len(Dshape),len(Dcrh),len(Ddiameter),len(Dlength),
# len(Dweight),len(Zspeed),len(Zangle),len(Btype),len(Bthickness),len(Bstrength),len(Bdensity),
# len(Bdensity),len(Bratio),len(Xdepth),len(Xpenetrate),len(F)]
#
# length.sort(reverse=True)
# lastLength = length[0]
# dataAll = []
#
# for i in range(0,lastLength):
# data={
# "靶标类型": "",
# "弹体材料": "",
# "弹体材料强度": "",
# "弹头形状": "",
# " CRH ": "",
# "弹体直径": "",
# "弹体长度": "",
# "弹体质量": "",
# "着靶速度": "",
# "命中角": "",
# "靶标材料种类": "",
# "靶标厚度": "",
# "靶标抗压强度": "",
# "靶标材料密度": "",
# "靶标配筋率": "",
# "侵彻深度": "",
# "贯穿": "",
# "发射炮类型": ""
# }
# if len(Dtype)==1:
# data["靶标类型"] = Dtype[0]
# elif len(Dtype)>1:
# data["靶标类型"] = Dtype[i]
# if len(Zspeed)==1:
# data["着靶速度"] = Zspeed[0]
# elif len(Zspeed)>1:
# data["着靶速度"] = Zspeed[i]
# if len(Zangle)==1:
# data["命中角"] = Zangle[0]
# elif len(Zangle)>1:
# data["命中角"] = Zangle[i]
# dataAll.append(data)
# print(".................")
# print(Zspeed)
# print(Zangle)
dataLast = {'data': TextAll}
return HttpResponse(json.dumps(dataLast, ensure_ascii=False),
content_type="application/json;charset=utf-8")
def run_rnn(n_outputs: int,
x_train: np.ndarray,
y_train: np.ndarray,
n_features=1,
epochs=10,
learning_rate=0.001):
"""
:param learning_rate:
:param n_outputs: number of categories
:param x_train: in the format [samples, time steps, features]
:param y_train: one hot encoded with keras.utils.to_categorical
:param n_features: number of feature
:param epochs:
:return: void
"""
n_neurons = 100
cells = [
keras.layers.LSTMCell(n_neurons),
keras.layers.LSTMCell(n_neurons),
keras.layers.LSTMCell(n_neurons),
]
model = Sequential([
# RNN(cells, input_shape=(None, n_features)),
#GRU(n_neurons, input_shape=(None, n_features)),
Masking(mask_value=0, input_shape=(None, n_features)),
# SimpleRNN(n_neurons, input_shape=(None, n_features), return_state=True),
# SimpleRNN(n_neurons),
GRU(n_neurons),
Dropout(0.5),
Dense(n_outputs),
Activation('softmax')
])
adam = keras.optimizers.adam(lr=learning_rate)
model.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
board_callback = keras.callbacks.TensorBoard(log_dir='./logs',
histogram_freq=1,
batch_size=32,
write_graph=True,
write_grads=True,
write_images=True,
embeddings_freq=0,
embeddings_layer_names=True,
embeddings_metadata=True)
print(model.summary())
history = model.fit(x_train,
y_train,
epochs=epochs,
batch_size=100,
validation_split=0.33,
verbose=2,
callbacks=[board_callback])
# list all data in history
print(history.history.keys())
# summarize history for accuracy
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
score = model.evaluate(x_train, y_train, batch_size=128)
print(score)
return model
# Filterbanks
x = Lambda(lambda x: K.log(x + 0.01), name='fbank_lognorm')(output_fft)
x = Dense(512, activation='relu', name='fbank_1')(x)
x = Dense(512, activation='relu', name='fbank_2')(x)
output_logfbank = Dense(26, activation='linear', name='fbank_3')(x)
# DCT
x = DCTLayer(numcep=13, norm='ortho', trainable=False, input_shape=(26,), name='dct')(output_logfbank)
# Lifter
x = Dense(512, activation='relu', name='lifter_1')(x)
x = Dense(512, activation='relu', name='lifter_2')(x)
output_lifter = Dense(12, activation='linear', name='lifter_3')(x)
# Log Energy
x = Dense(1, activation='linear', input_dim=257, kernel_initializer='ones', bias_initializer='zeros',
trainable=False, name='sum_energy')(output_fft)
output_energy = Lambda(lambda x: K.log(x), name='log_energy')(x)
output_mfcc = concatenate([output_energy, output_lifter])
return inputs, output_mfcc
def rnn_model(input=None, hidden_layers=5, dropout_rate=.5, bottleneck_size=125):
if input = None:
input = Input(shape=(13), name='classifier_input')
x = Masking(mask_value=0.)(input)
for i in range(hidden_layers-1):
x = LSTM(250, return_sequences=True)(x)
x = Dropout(dropout_rate)(x)
x = LSTM(bottleneck_size, return_sequences=True)(x)
x = Dropout(dropout_rate)(x)
output = TimeDistributed(Dense(39, activation='softmax'))(x)
return input, output
X_train = encode_data(X_train, vocab_size)
X_dev = encode_data(load_data(data_model, 'X_dev'), vocab_size)
X_test = encode_data(load_data(data_model, 'X_test'), vocab_size)
y_train = load_data(data_model, 'y_train')
y_dev = load_data(data_model, 'y_dev')
y_test = load_data(data_model, 'y_test')
print('Loaded data.')
# In[63]:
model = Sequential()
model.add(
Masking(mask_value=0.,
input_shape=(seq_length, vocab_size),
name='Masking'))
model.add(LSTM(output_dim=lstm_size, dropout_U=0.2, dropout_W=0.2,
name='LSTM'))
model.add(Dense(output_dim=4, activation='softmax', name='Softmax'))
# In[64]:
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print('Compiled model.')
print(model.summary())
def trainNetwork(parameters, x1t, x1v, x2t, x2v, yt, yv):
# Hyperparameters
learning_rate = parameters['learning_rate']
output_lstm = parameters['lstm']
layers = parameters['n_layers']
dense_nodes = parameters['dense_nodes']
print_layers = [0] * max_layers
for i, v in enumerate(dense_nodes):
print_layers[i] = v
validation_size = x1v.shape[0]
dense_initializer = keras.initializers.RandomNormal(seed=7051)
lstm_kernel_initializer = keras.initializers.glorot_uniform(seed=7051)
lstm_recurrent_initializer = keras.initializers.orthogonal(seed=7051)
# Input definition
input_eular = Input(shape=(n_joints, 1),
dtype='float32',
name='input_eular')
input_crp = Input(shape=(1, ), dtype='float32', name='input_crp')
# Model definition
x = Masking(mask_value=-1)(input_eular)
x = LSTM(output_lstm,
return_sequences=False,
kernel_initializer=lstm_kernel_initializer,
recurrent_initializer=lstm_recurrent_initializer)(x)
x = concatenate([x, input_crp])
for nodes in dense_nodes:
x = Dense(nodes,
activation='relu',
kernel_initializer=dense_initializer)(x)
output = Dense(1)(x)
model = Model(input=[input_eular, input_crp], output=output)
optimizer = Adam(lr=learning_rate)
model.compile(optimizer=optimizer, loss='mse', metrics=['mae'])
hist = model.fit([x1t, x2t],
yt,
validation_data=([x1v, x2v], yv),
epochs=epochs,
batch_size=batch_size,
verbose=False)
head = [learning_rate]
for l in print_layers:
head.append(l)
head.append(validation_size)
mae = hist.history['val_mean_absolute_error']
mse = hist.history['val_loss']
t_mse = hist.history['loss']
print_list = []
print_list += head
print_list.append(':')
print_list.append('val_mae')
print_list += mae
print_list.append(':')
print_list.append('val_mse')
print_list += mse
print_list.append(':')
print_list.append('train_mse')
print_list += t_mse
print("{:.2f}".format(np.min(mse)), "-", head)
with open('results.csv', mode='a') as csv_file:
writer = csv.writer(csv_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
lineterminator='\n')
writer.writerow(print_list)
keras.backend.clear_session()
return mse
def prot_encoder(model, seq, seq_size):
encoder = model.prot_encoder
if model.dataset.name in LIST_AA_DATASETS:
if encoder['conv_strides'] != 1 or encoder['name'] == 'conv_biLSTM_att'\
or model.batch_size != 1:
print('!! ERROR !!: AA prediction but conv_strides > 1 ' +
'OR tt_mech at the end of encoder OR batch_size > 1 !!')
exit(1)
kreg, breg = None, None
# if model.prot_reg != 0:
# kreg, breg = regularizers.l2(model.prot_reg), regularizers.l2(model.prot_reg)
# else:
# kreg, breg = None, None
seq = Lambda(lambda t: expand_last_dim(t), name="expand_last_dim")(seq)
if encoder['name'] == 'conv' or encoder['name'] == 'conv_aa':
n_steps = encoder['n_steps']
n_filters = encoder['nb_conv_filters']
# seq = Lambda(lambda t: t, name='temp1')(seq)
for n in range(n_steps):
seq_shape = seq.get_shape().as_list()
kernel_size = (seq_shape[1], encoder['filter_size'])
strides = (seq_shape[1], encoder['conv_strides'])
seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg,
breg, model.prot_reg, model.prot_dropout,
model.prot_BN, n)
print('seq', seq)
seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))(
seq_size)
# seq = Lambda(lambda t: t, name='temp2')(seq)
if model.dataset.name not in LIST_AA_DATASETS and 'aa' not in encoder[
'name']:
embedding = Lambda(lambda t: agg_sum(t, 2), name='agg_sum')(seq)
print('###', embedding)
embedding = Lambda(lambda t: squeeze(t, -1),
name='prot_embedding')(embedding)
elif 'aa' in encoder['name']:
seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq)
seq = Permute((2, 1))(seq)
embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
else:
seq = Permute((2, 1, 3))(seq)
embedding = Lambda(lambda t: squeeze(squeeze(t, -1), 0),
name='prot_embedding')(seq)
elif 'conv_biLSTM' in encoder['name'] or 'conv_biGRU' in encoder['name']:
seq_shape = seq.get_shape().as_list()
n_filters = encoder['nb_conv_filters']
kernel_size = (seq_shape[1], encoder['filter_size'])
strides = (seq_shape[1], encoder['conv_strides'])
seq_size = Lambda(lambda t: update_seq_size(t, float(strides[1])))(
seq_size)
seq = seq_conv(model, seq, n_filters, kernel_size, strides, kreg, breg,
model.prot_reg, model.prot_dropout, model.prot_BN, 0)
seq = Lambda(lambda t: squeeze(t, -1), name='conv_emb')(seq)
seq = Permute((2, 1))(seq)
if model.dataset.name not in LIST_AA_DATASETS:
seq = Masking(mask_value=0.0)(seq)
rseq = True if 'att' not in encoder['name'] and 'aa' not in encoder[
'name'] else True
if 'biLSTM' in encoder['name']:
lstm_layer = LSTM(n_filters,
return_sequences=rseq,
activation='tanh',
kernel_regularizer=kreg,
recurrent_regularizer=kreg,
bias_regularizer=breg,
dropout=model.prot_dropout)
elif 'biGRU' in encoder['name']:
lstm_layer = GRU(n_filters,
return_sequences=rseq,
activation='tanh',
kernel_regularizer=kreg,
recurrent_regularizer=kreg,
bias_regularizer=breg,
dropout=model.prot_dropout)
# import pdb; pdb.Pdb().set_trace()
seq = Bidirectional(lstm_layer,
merge_mode='concat',
weights=None,
name='biLSTM')(seq)
if model.prot_reg != 0:
seq = GaussianNoise(model.prot_reg)(seq)
if 'att' in encoder['name']:
attention_probs = Dense(n_filters * 2,
activation='softmax',
name='attention_probs')(seq)
embedding = Multiply(name='prot_embedding')([seq, attention_probs])
elif model.dataset.name in LIST_AA_DATASETS or 'aa' in encoder['name']:
embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
else:
embedding = Lambda(lambda t: agg_sum(t, 1),
name='prot_embedding')(seq)
# embedding = Lambda(lambda t: t, name='prot_embedding')(seq)
# seq = Permute((2, 1))(seq)
# import pdb; pdb.Pdb().set_trace()
# embedding = Lambda(lambda t: squeeze(t, 0), name='prot_embedding')(seq)
if encoder['hand_crafted_features']:
embedding = Concatenate(axis=-1)([embedding, model.features])
# import pdb; pdb.Pdb().set_trace()
return embedding, seq_size
def build_model(mode='bbox'):
print('mode:', mode)
unit_size = 100
regions_size = 7 * 7
beta_size = 2 + 1 + 1 # 2 objects + 1 sentential + 1 spatial
delayed_sent = Input(shape=[max_len + 1])
sf_size = 11 # dx; dy; ov; ov1; ov2; h1; w1; h2; w2; a1; a2 (from VisKE)
beta_feature_size = 2 * (beta_size - 1) * unit_size
if mode[:9] == 'attention':
sf_size = 49 * 2 # attention mask pattern
beta_feature_size = 2 * (beta_size - 1) * unit_size
elif mode[:4] == 'bbox':
beta_feature_size = 2 * (beta_size - 1) * unit_size
elif mode[:8] == 'implicit':
beta_size = 2 + 1
beta_feature_size = 2 * (beta_size - 1) * unit_size
elif mode == 'spatial_adaptive':
beta_size = regions_size + 1
beta_feature_size = 2 * (beta_size - 1) * unit_size
elif mode == 'spatial_adaptive-bbox':
beta_size = regions_size + 1 + 1
beta_feature_size = 2 * (beta_size - 1) * unit_size
elif mode == 'spatial_adaptive-attention':
sf_size = 49
beta_size = regions_size + 1 + 1
beta_feature_size = 2 * (beta_size - 2) * unit_size
visual_features_in0 = Input(shape=[regions_size, 2048]) # resnet50
visual_features_objs_in = Input(shape=[2, 2048]) # resnet50
spatial_features_in = Input(shape=[sf_size])
embeddings = Embedding(len(word2ix), unit_size)(Masking()(delayed_sent))
# fine tune / project features
mlp_vision = Dense(unit_size, activation='relu')
mlp_space = Sequential([
Dense(unit_size, activation='tanh', input_shape=[sf_size]),
Dense(unit_size, activation='relu'),
])
mlp_att = Sequential([
TimeDistributed(Dense(unit_size, activation='relu'),
input_shape=[max_len + 1, beta_feature_size]),
TimeDistributed(Dense(unit_size, activation='tanh')),
TimeDistributed(Dense(beta_size, activation='softmax')),
])
### global visual features
visual_features0 = TimeDistributed(mlp_vision)(
visual_features_in0) # learn to find
visual_features0_g = Reshape([7, 7, unit_size])(visual_features0)
visual_features0_g = Flatten()(AveragePooling2D([7,
7])(visual_features0_g))
### objects visual features
visual_features_objs = TimeDistributed(mlp_vision)(visual_features_objs_in)
spatial_features = mlp_space(spatial_features_in)
### adaptive attention: beta (which feature set needs more attention?)
def feature_fusion(x, regions_size=regions_size, max_len=max_len):
return K.concatenate([
x[0],
K.repeat_elements(K.expand_dims(x[1], 1), max_len + 1, 1),
], 2)
def beta_features(x,
unit_size=unit_size,
max_len=max_len,
beta_size=beta_size):
if mode[:
8] == 'implicit' or mode == 'spatial_adaptive' or mode == 'spatial_adaptive-attention':
h, vf0 = x
vf0_ = K.repeat_elements(
K.expand_dims(vf0, 1), max_len + 1,
1) # [sent, 49, unit_size] or [sent, 2, unit_size]
if mode == 'spatial_adaptive-attention':
h_ = K.repeat_elements(K.expand_dims(h, 2), beta_size - 2,
2) # [sent, 49, unit_size]
return K.reshape(
K.concatenate([h_, vf0_], 3),
[-1, max_len + 1, 2 *
(beta_size - 2) * unit_size]) # [sent, 49*b*unit_size]
else:
h_ = K.repeat_elements(K.expand_dims(h, 2), beta_size - 1,
2) # [sent, 49, unit_size]
return K.reshape(
K.concatenate([h_, vf0_], 3),
[-1, max_len + 1, 2 *
(beta_size - 1) * unit_size]) # [sent, 49*b*unit_size]
else:
h, sf, vf0 = x
sf_ = K.expand_dims(
K.repeat_elements(K.expand_dims(sf, 1), max_len + 1, 1),
2) # [sent, 1, unit_size]
vf0_ = K.repeat_elements(
K.expand_dims(vf0, 1), max_len + 1,
1) # [sent, 49, unit_size] or [sent, 2, unit_size]
vf_sf = K.concatenate(
[sf_, vf0_],
2) # [sent, 49+1, unit_size] or [sent, 2+1, unit_size]
h_ = K.repeat_elements(
K.expand_dims(h, 2), beta_size - 1,
2) # [sent, 49+1, unit_size] or or [sent, 2+1, unit_size]
return K.reshape(
K.concatenate([h_, vf_sf], 3),
[-1, max_len + 1, 2 *
(beta_size - 1) * unit_size]) # [sent, 49+1*b*unit_size]
### use adaptive attention beta
def adaptation_attention(x,
max_len=max_len,
regions_size=regions_size,
mode=mode):
if mode[:8] == 'implicit' or mode == 'spatial_adaptive':
h, vf0, b = x
vf0_ = K.repeat_elements(K.expand_dims(vf0, 1), max_len + 1, 1)
return b[:, :, 0:1] * h + K.sum(
K.expand_dims(b[:, :, 1:], 3) * vf0_, 2)
else:
h, sf, vf0, b = x
if len(sf.get_shape()) == 2:
sf_ = K.repeat_elements(K.expand_dims(sf, 1), max_len + 1, 1)
else:
sf_ = sf
vf0_ = K.repeat_elements(K.expand_dims(vf0, 1), max_len + 1, 1)
return b[:, :, 0:1] * h + b[:, :, 1:2] * sf_ + K.sum(
K.expand_dims(b[:, :, 2:], 3) * vf0_, 2)
fused_features = Lambda(feature_fusion)([embeddings, visual_features0_g])
hidden_a = LSTM(unit_size, return_sequences=True)(fused_features)
ling_features = LSTM(unit_size, return_sequences=True)(hidden_a)
if mode[:4] == 'bbox' or mode[:9] == 'attention':
beta_feaures_out = Lambda(beta_features)(
[hidden_a, spatial_features, visual_features_objs])
beta = mlp_att(beta_feaures_out)
adapted_feaures = Lambda(adaptation_attention)(
[ling_features, spatial_features, visual_features_objs, beta])
out = Dense(len(word2ix), activation='softmax')(adapted_feaures)
model = Model([
visual_features_in0, spatial_features_in, visual_features_objs_in,
delayed_sent
], out)
elif mode[:8] == 'implicit':
beta_feaures_out = Lambda(beta_features)(
[hidden_a, visual_features_objs])
beta = mlp_att(beta_feaures_out)
adapted_feaures = Lambda(adaptation_attention)(
[ling_features, visual_features_objs, beta])
out = Dense(len(word2ix), activation='softmax')(adapted_feaures)
model = Model(
[visual_features_in0, visual_features_objs_in, delayed_sent], out)
elif mode[:7] == 'no-beta':
out = Dense(len(word2ix), activation='softmax')(fused_features)
model = Model([visual_features_in0, delayed_sent], out)
elif mode == 'spatial_adaptive':
beta_feaures_out = Lambda(beta_features)([hidden_a, visual_features0])
beta = mlp_att(beta_feaures_out)
adapted_feaures = Lambda(adaptation_attention)(
[ling_features, visual_features0, beta])
out = Dense(len(word2ix), activation='softmax')(adapted_feaures)
model = Model([visual_features_in0, delayed_sent], out)
elif mode == 'spatial_adaptive-bbox':
beta_feaures_out = Lambda(beta_features)(
[hidden_a, spatial_features, visual_features0])
beta = mlp_att(beta_feaures_out)
adapted_feaures = Lambda(adaptation_attention)(
[ling_features, spatial_features, visual_features0, beta])
out = Dense(len(word2ix), activation='softmax')(adapted_feaures)
model = Model([visual_features_in0, spatial_features_in, delayed_sent],
out)
elif mode == 'spatial_adaptive-attention':
beta_feaures_out = Lambda(beta_features)([hidden_a, visual_features0])
beta = mlp_att(beta_feaures_out)
beta_spatial = Lambda(lambda x: x[:, :, 2:])(beta)
spatial_features = mlp_space(beta_spatial)
adapted_feaures = Lambda(adaptation_attention)(
[ling_features, spatial_features, visual_features0, beta])
out = Dense(len(word2ix), activation='softmax')(adapted_feaures)
model = Model([visual_features_in0, delayed_sent], out)
model.summary()
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')
return model
def get_gru_baseline(self):
lstm_qo = GRU(100, return_sequences=False)
get_diag = Lambda(
lambda xin: K.sum(xin * T.eye(self.max_opt_count), axis=2),
output_shape=(self.max_opt_count, ))
transp_out = Lambda(lambda xin: K.permute_dimensions(xin, (0, 2, 1)),
output_shape=(self.max_opt_count, 100))
apply_weights = Lambda(lambda xin: (K.expand_dims(xin[
0], axis=-1) * K.expand_dims(xin[1], axis=2)).sum(axis=1),
output_shape=(100, self.max_opt_count))
tile_q = Lambda(lambda xin: K.tile(xin, (1, self.max_opt_count, 1, 1)),
output_shape=(self.max_opt_count, self.max_q_length,
self.word_vec_size))
exp_dims = Lambda(lambda xin: K.expand_dims(xin, 1),
output_shape=(1, self.max_q_length,
self.word_vec_size))
exp_layer = Lambda(lambda xin: K.exp(xin),
output_shape=(self.max_sent_para,
self.max_opt_count))
mask_weights = Lambda(lambda xin: T.switch(T.eq(xin, 0), np.NINF, xin),
output_shape=(self.max_sent_para,
self.max_opt_count))
final_weights = Lambda(lambda xin: xin / K.cast(
K.sum(xin, axis=1, keepdims=True), K.floatx()),
output_shape=(self.max_sent_para,
self.max_opt_count))
q_input = Input(shape=(self.max_q_length, self.word_vec_size),
name='question_input')
q_exp = exp_dims(q_input)
q_rep = tile_q(q_exp)
option_input = Input(shape=(self.max_opt_count, self.max_option_length,
self.word_vec_size),
name='option_input')
opt_q = Concatenate(axis=2)([q_rep, option_input])
lstm_input = Input(shape=(None, self.word_vec_size), name='lstm_input')
lstm_mask = Masking(mask_value=0.)(lstm_input)
lstm_out = lstm_qo(lstm_mask)
lstm_model = Model(inputs=lstm_input, outputs=lstm_out)
lstm_td_opt = TimeDistributed(lstm_model)(opt_q)
doc_input = Input(shape=(self.max_sent_para, self.max_words_sent,
self.word_vec_size),
name='doc_input')
lstm_doc = TimeDistributed(lstm_model)(doc_input)
att_wts = Dot(axes=2, normalize=True)([lstm_doc, lstm_td_opt])
att_wts = mask_weights(att_wts)
att_wts = exp_layer(att_wts)
att_wts = final_weights(att_wts)
out = apply_weights([lstm_doc, att_wts])
out = transp_out(out)
dp = Dot(axes=2, normalize=True)([out, lstm_td_opt])
out = get_diag(dp)
probs = MaskedSoftmax()([out, option_input])
main_model = Model(inputs=[q_input, doc_input, option_input],
outputs=probs)
sgd = SGD(lr=0.1, decay=0., momentum=0., nesterov=False)
main_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
main_model.summary()
return main_model
y_one_hot = np.zeros((datapoints, seq_len, num_classes))
for i in range(datapoints):
for j in range(seq_len):
y_one_hot[i][j][y[i][j]] = 1
embedding_size = 10
embedding_weights = np.zeros((num_symbols, embedding_size))
for i in range(num_symbols):
embedding_weights[i] = np.random.rand(embedding_size)
input_layer = Input(shape=(seq_len, ), dtype=np.int32)
embedding = Embedding(num_symbols, embedding_size, input_length=seq_len)
embedded_input = embedding(input_layer)
mask = Masking(mask_value=0)(embedded_input)
bidirect = Bidirectional(LSTM(100, return_sequences=True))(mask)
final = TimeDistributed(Dense(num_classes, activation="softmax"))(bidirect)
model = Model(inputs=[input_layer], outputs=[final])
model.compile(loss="categorical_crossentropy",
optimizer="rmsprop",
metrics=["accuracy"])
model.fit(x, y_one_hot)
print(x)
print(y)
print(y_one_hot)
print(model.predict(x))
model.save("issue_3676.model")
for k in range(n_folds):
# Load data for this fold
data = np.load(dataset_path_prefix + f'{k}fold_{dataset_type}.npz')
MASK_VALUE = data['MASK_VALUE']
X_train, Y_train = data['X_train'], data['Y_train']
X_val, Y_val = data['X_val'], data['Y_val']
# X_test, Y_test = data['X_test'], data['Y_test']
#######################################################################################################
# Create model
model = Sequential()
model.add(
Masking(mask_value=MASK_VALUE,
input_shape=(WINDOW_SIZE, N_FEATURES)))
for n_units in GRU_ARCH:
model.add(GRU(n_units, return_sequences=True,
dropout=0.1)) #, recurrent_dropout=0.2))
model.add(TimeDistributed(Dense(1, activation='sigmoid')))
model.compile(loss='binary_crossentropy', optimizer='adam')
if k == 0:
print(model.summary())
#######################################################################################################
# Train
n_epochs = 100
batch_size = 128
# Set early stopping
metric_a = Input(shape=(12, ), name='metric_a')
# 输入3：交互信息
# 度量值和文本交互
# att_a = Input(shape=(MAX_SEQUENCE_LENGTH + 12, EMBEDDING_DIM), name='interaction_a') # 212*200
# att_input = Input(shape=(200 + 12, 200))
# 文本之间交互
att_b = Input(shape=(MAX_SEQUENCE_LENGTH, EMBEDDING_DIM), name='interaction_b')
# 度量值之间交互
# att_c = Input(shape=(3, EMBEDDING_DIM), name='interaction_c')
# att_c1 = Input(shape=(3, EMBEDDING_DIM), name='interaction_c1')
# att_c2 = Input(shape=(3, EMBEDDING_DIM), name='interaction_c2')
# att_c3 = Input(shape=(6, EMBEDDING_DIM), name='interaction_c3')
# 处理文本信息：LSTM
masking_layer = Masking(mask_value=0,
input_shape=(MAX_SEQUENCE_LENGTH, EMBEDDING_DIM))
# LSTM层 输出维度为2，激活函数为sigmoid，并对该层权重做均匀分布(uniform)初始化
lstm_share = LSTM(output_dim=2, activation='sigmoid', init='uniform')
embedding_a = masking_layer(method_a) # 嵌入层
lstm_a = lstm_share(embedding_a)
# 处理度量值信息：直接Dense
dense_share2 = Dense(12, activation='tanh', init='uniform') # 全连接层
mtrdense_a = dense_share2(metric_a)
# 处理交互信息：
def build_interation(att, filters, units):
# 构建3层interacting layer
for _ in range(3):
def masking(layer, layer_in, layerId, tensor=True):
out = {layerId: Masking(mask_value=layer['params']['mask_value'])}
if tensor:
out[layerId] = out[layerId](*layer_in)
return out
F2predict = {} #saves the softmax output
F2test = {} #saves the test set GTs
F2_modelpred = {} #saves the final outputs
F2con = {} #saves the confusion matrix per speaker
F2ACC = {} #saves the accuracy per speaker
F2WR = {} #saves the weighted accuracy per speaker
F2UWR = {} #saves the unweighted recall per speaker
F2VAL_ACC = {} #saves valence accuracies
F2TR_ACC = {} #saves the training accuracy
F = 0
for train, test in logo.split(A, Y, grp):
label_train = np_utils.to_categorical(Y[train])
# Set callback functions to early stop training and save the best model so far
callbacks = [EarlyStopping(monitor='val_loss', patience=10)]
model = Sequential()
model.add(Masking(mask_value=0., input_shape=(W_Length, 895)))
model.add(Bidirectional(LSTM(128, return_sequences=True)))
model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(128)))
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5)) ##
model.add(Dense(4, activation='softmax'))
adam = keras.optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
sgd = keras.optimizers.SGD(lr=0.001,
momentum=0.0,
print('padding sequences')
train_datas = sequence.pad_sequences(train_datas, maxlen=MAX_LEN)
train_labels = sequence.pad_sequences(train_labels, maxlen=MAX_LEN)
test_datas = sequence.pad_sequences(test_datas, maxlen=MAX_LEN)
test_labels = sequence.pad_sequences(test_labels, maxlen=MAX_LEN)
print('x_train shape:', train_datas.shape)
print('x_test shape:', test_datas.shape)
train_labels = keras.utils.to_categorical(train_labels, CLASS_NUMS)
test_labels = keras.utils.to_categorical(test_labels, CLASS_NUMS)
print('trainlabels shape:', train_labels.shape)
print('testlabels shape:', test_labels.shape)
## BiLSTM+CRF模型构建
inputs = Input(shape=(MAX_LEN, ), dtype='int32')
x = Masking(mask_value=0)(inputs)
x = Embedding(VOCAB_SIZE, EMBED_DIM, mask_zero=True)(x)
x = Bidirectional(LSTM(HIDDEN_SIZE, return_sequences=True))(x)
x = TimeDistributed(Dense(CLASS_NUMS))(x)
outputs = CRF(CLASS_NUMS)(x)
model = Model(inputs=inputs, outputs=outputs)
model.summary()
import numpy as np
from keras.callbacks import Callback
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score
from keras.callbacks import Callback
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score
def masking(layer, layer_in, layerId):
out = {layerId: Masking(mask_value=layer['params']['mask_value'])(*layer_in)}
return out
def modelrun(modelfrom, predictfrom):
"""
"""
# We need to address the missing values (-999s) before scaling.
# Create masks of the modelfrom and predictfrom
modelfrommask = np.ma.masked_equal(modelfrom, -999).mask
predictfrommask = np.ma.masked_equal(predictfrom, -999).mask
# Use them to reassign -999s as max stat value
modelfrom[modelfrommask] = (np.ones_like(modelfrom) *
np.max(modelfrom, (0, 1)))[modelfrommask]
predictfrom[predictfrommask] = (np.ones_like(predictfrom) *
np.max(predictfrom,
(0, 1)))[predictfrommask]
# Apply the 3D scaler:
scaler = MinMaxScaler(feature_range=(0, 1))
# Design the scaler:
# (this flattens the 3D array into 2D, applies determines scaler, then re-stacks in 3D)
scaler = scaler.fit(modelfrom.reshape(-1, modelfrom.shape[2]))
# Apply the scaler:
modelfrom_scaled = scaler.transform(
modelfrom.reshape(-1, modelfrom.shape[2])).reshape(modelfrom.shape)
predictfrom_scaled = scaler.transform(
predictfrom.reshape(-1,
predictfrom.shape[2])).reshape(predictfrom.shape)
# Return the missing values to -999
modelfrom[modelfrommask] = -999
predictfrom[predictfrommask] = -999
# Split into test and training sets:
train, test = train_test_split(modelfrom_scaled, test_size=0.1)
# Split into independant and responding variables:
# Split the training data into independant and responding variables:
train_ind, train_resp = train[:, :, :-1], train[:, :, -1]
# Split test data:
test_ind, test_resp = test[:, :, :-1], test[:, :, -1]
# Split prediction data:
predictfrom_ind, predictfrom_resp = predictfrom_scaled[:, :, :
-1], predictfrom_scaled[:, :,
-1]
#Design and train the LSTM model:
# Design LSTM neural network
# Define the network using the Sequential Keras API
model = Sequential()
# Inform algorithm that 0 represents non-values (values of -1 were scaled to 0!)
model.add(
Masking(mask_value=-999,
input_shape=(train_ind.shape[1], train_ind.shape[2])))
# Define as LSTM with 8 neurons - not optimized - use 8 because I have 8 statistical categories
model.add(LSTM(6, return_sequences=True))
model.add(LSTM(6, return_sequences=True))
model.add(LSTM(6, return_sequences=True))
model.add(LSTM(6, return_sequences=True))
model.add(LSTM(6))
# I'm not even sure why I need this part, but it doesn't work without it...
model.add(Dense(train_ind.shape[1]))
# Define a loss function and the Adam optimization algorithm
model.compile(loss='mean_squared_error', optimizer='adam')
# train network
history = model.fit(train_ind,
train_resp,
epochs=50,
batch_size=5,
validation_data=(test_ind, test_resp),
verbose=0,
shuffle=False)
# plot history
# plt.plot(history.history['loss'], label='train')
# plt.plot(history.history['val_loss'], label='test')
# plt.xlabel('Epoch')
# plt.ylabel('Loss')
# plt.legend()
# plt.show()
# Make a prediction:
predicted_resp = model.predict(predictfrom_ind)
# Invert scaling:
# Make prediced_resp dimension match predictfrom_ind
predicted_resp = np.expand_dims(predicted_resp, axis=2)
# Invert scaling for forecast
# Add the predicted values to the independent variables used for the prediction
inv_predicted = np.concatenate((predictfrom_ind[:, :, :], predicted_resp),
axis=2)
inv_predicted = scaler.inverse_transform(
inv_predicted.reshape(-1, inv_predicted.shape[2])).reshape(
inv_predicted.shape)
# Make sure the missing data is ignored
test_predicted = np.empty_like(inv_predicted)
test_predicted[~predictfrommask] = inv_predicted[~predictfrommask]
# Isolate the predicted values
inv_predicted_resp = test_predicted[:, :, -1]
# Return results (predicted responding variables):
return inv_predicted_resp
def main(MODEL_FILE):
print "Loading hdf5's..."
test_dict = io.load('./data/test_dict_IPConv_ntuple_'+ RUN_NAME +'.h5')
train_dict = io.load('./data/train_dict_IPConv_ntuple_'+ RUN_NAME +'.h5')
X_train = train_dict['X']
y_train = train_dict['y']
X_test = test_dict['X']
y_test = test_dict['y']
n_features = X_test.shape[2]
# this is a df
ip3d = test_dict['ip3d']
print 'Building model...'
if (MODEL_FILE == 'CRNN'):
graph = build_graph(n_features)
model = Sequential()
model.add(graph)
# remove Maxout for tensorflow
model.add(MaxoutDense(64, 5, input_shape=graph.nodes['dropout'].output_shape[1:]))
model.add(Dense(64))
elif (MODEL_FILE == 'RNN'):
model = Sequential()
model.add(Masking(mask_value=-999, input_shape=(N_TRACKS, n_features)))
model.add(GRU(25))#, input_shape=(N_TRACKS, n_features))) #GRU
model.add(Dropout(0.2)) #0.2
# remove Maxout for tensorflow
model.add(MaxoutDense(64, 5)) #, input_shape=graph.nodes['dropout'].output_shape[1:]))
model.add(Dense(64))
model.add(Dropout(0.4))
model.add(Highway(activation = 'relu'))
model.add(Dropout(0.3))
model.add(Dense(4))
model.add(Activation('softmax'))
print 'Compiling model...'
model.compile('adam', 'categorical_crossentropy')
model.summary()
print 'Training:'
try:
model.fit(X_train, y_train, batch_size=512,
callbacks = [
EarlyStopping(verbose=True, patience=20, monitor='val_loss'),
ModelCheckpoint(MODEL_FILE + RUN_NAME +'-progress', monitor='val_loss', verbose=True, save_best_only=True)
],
nb_epoch=100,
validation_split = 0.2,
show_accuracy=True)
except KeyboardInterrupt:
print 'Training ended early.'
# -- load in best network
model.load_weights(MODEL_FILE + RUN_NAME +'-progress')
if (SAVE_PROTOBUF):
print 'Saving protobuf'
# write out to a new directory called models
# the actual graph file is graph.pb
# the graph def is in the global session
import tensorflow as tf
import keras.backend.tensorflow_backend as tfbe
sess = tfbe._SESSION
saver = tf.train.Saver()
tf.train.write_graph(sess.graph_def, 'models/', 'graph.pb', as_text=False)
save_path = saver.save(sess, "./model-weights.ckpt")
print "Model saved in file: %s" % save_path
print saver.as_saver_def().filename_tensor_name
print saver.as_saver_def().restore_op_name
print model.get_output()
print 'Saving weights...'
model.save_weights('./weights/ip3d-replacement_' + MODEL_FILE + RUN_NAME +'.h5', overwrite=True)
json_string = model.to_json()
open(MODEL_FILE + RUN_NAME +'.json', 'w').write(json_string)
print 'Testing...'
yhat = model.predict(X_test, verbose = True, batch_size = 512)
io.save('yhat'+ RUN_NAME +'.h5', yhat)
print 'Plotting ROC...'
fg = plot_ROC(y_test, yhat, ip3d, MODEL_FILE)
#plt.show()
fg.savefig('./plots/roc' + MODEL_FILE + RUN_NAME +'.pdf')
# x_test = np.array( pd.read_csv("x_test.csv",header=None) )
# x_test = np.reshape(x_test,(x_test.shape[0],7,1))
"""
上面是文件格式
下面是一个个学生，list格式
"""
x_test = np.array([[78.86,74.89,-2,-2,-2,-2,-2],[78.17,80.13,82.53,82.76,53.22,-2,-2]])
x_test = np.reshape(x_test,(x_test.shape[0],7,1))
batch_size = 32 #超参
epochs = 1000 #超参
units = 6 #超参 4不行
model = Sequential()
model.add(Masking(mask_value=-2., input_shape=(7,1)))
model.add(LSTM(units))
model.add(Dense(1))
print(model.summary())
model.compile(loss='mean_squared_error', optimizer='adam',metrics=['mse', 'mape'])
# filepath = './lstmfc/model-ep{epoch:03d}-mse{mean_squared_error:.3f}-val_mse{val_mean_squared_error:.3f}-val_mape{val_mean_absolute_percentage_error}.h5'
# checkpoint = keras.callbacks.ModelCheckpoint(filepath, monitor='val_mean_squared_error', verbose=1, save_best_only=True, mode='min')
# model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1, validation_data=(x_test, y_test), shuffle=True, callbacks=[checkpoint])
###predict
model.load_weights('./lstmfc/model-ep995-mse28.667-val_mse28.815-val_mape4.917600361394993.h5')
print('load weights...')
reeee = model.predict(x_test)
print(reeee)
print(reeee.shape)
def build_model(params, num_features):
num_lstm_units = int(params['num_lstm_units'])
num_lstm_layers = int(params['num_lstm_layers'])
num_dense_layers = int(params['num_dense_layers'])
num_dense_units = int(params['num_dense_units'])
l2reg = params['l2reg']
inputs_ab = Input(shape=(None, num_features))
mask_ab = Masking(mask_value=0.)(inputs_ab)
lstm_abc = LSTM(num_lstm_units, return_sequences=True)(mask_ab)
for nl in range(num_lstm_layers - 1):
lstm_ab2 = LSTM(num_lstm_units, return_sequences=True)(lstm_abc)
lstm_abc = lstm_ab2
# lstma =LSTM(num_lstm_units, return_sequences=True)(lstm_abc)
hidden1_a = TimeDistributed(Dense(num_dense_units))(lstm_abc)
for nl in range(num_dense_layers - 1):
hidden2_a = TimeDistributed(Dense(num_dense_units))(hidden1_a)
hidden1_a = hidden2_a
dropout_a = Dropout(0.5)(hidden1_a)
softmax_a = TimeDistributed(Dense(
4, activation='softmax', activity_regularizer=regularizers.l2(l2reg)),
name='softmaxa')(dropout_a)
lstm_b = LSTM(num_lstm_units, return_sequences=False)(lstm_abc)
hidden1_b = Dense(num_dense_units)(lstm_b)
for nl in range(num_dense_layers - 1):
hidden2_b = Dense(num_dense_units)(hidden1_b)
hidden1_b = hidden2_b
dropout_b = Dropout(0.5)(hidden1_b)
softmax_b = Dense(3,
activation='softmax',
activity_regularizer=regularizers.l2(l2reg),
name='softmaxb')(dropout_b)
lstm_c = LSTM(num_lstm_units, return_sequences=False)(lstm_abc)
hidden1_c = Dense(num_dense_units)(lstm_c)
for nl in range(num_dense_layers - 1):
hidden2_c = Dense(num_dense_units)(hidden1_c)
hidden1_c = hidden2_c
dropout_c = Dropout(0.5)(hidden1_c)
softmax_c = Dense(2,
activation='softmax',
activity_regularizer=regularizers.l2(l2reg),
name='softmaxc')(dropout_c)
model = Model(inputs=inputs_ab, outputs=[softmax_a, softmax_b, softmax_c])
model.compile(optimizer='adam',
loss={
'softmaxa': 'categorical_crossentropy',
'softmaxb': 'categorical_crossentropy',
'softmaxc': 'binary_crossentropy'
},
loss_weights={
'softmaxa': 0.3,
'softmaxb': 0.4,
'softmaxc': 0.3
},
metrics=['accuracy'],
sample_weight_mode={
'softmaxa': 'temporal',
'softmaxb': None,
'softmaxc': None
})
return model
# Recurrent means that the output at the current time step becomes input to next time step. Therefore it's not just the current input but what it remembers about the preceding element.
# Example: Considering the entire sentence as a whole rather than each individual word separately when forming a response.
# RNN should be able to see the words "But" and "Terrible Exciting" And realize that the meaning switches from negative to positive
# RNNs are made up of LSTM cells which maintain a cell state as well as a carry for ensuring that the signal (information in the form of a gradient) is not lost
# At each time step, LSTM cells consider the current word, the carry and the cell state.
# Tokenize (each word becomes its own feature)
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout, Masking, Embedding, CuDNNLSTM
model = Sequential()
model.add(
Embedding(input_dim=num_words,
input_length = training_length,
output_dim = 100,
weights = [embedding_matrix],
trainable = False,
mask_zero = True)
)
model.add(Masking(mask_value=0.0))
model.add(LSTM(64, return_sequences=False, dropout=0.1, recurrent_dropout=0.1))
model.add(Dense(64, activation='relu'))
index_pred = listwords.index(next_w)
Y_test[i, indseq, index_pred] = 1
indseq += 1 # Increment index if target label present in reduced dictionary
outfile = 'Testing_data_' + str(nbkeep)
np.savez(outfile, X_test=X_test, Y_test=Y_test) # Saving tensor
from keras import layers
from keras.layers import Dense, Activation, Masking
from keras.layers.recurrent import SimpleRNN
from keras.models import Sequential
from keras.optimizers import adam, RMSprop
model = Sequential()
model.add(Masking(mask_value=0., input_shape=(35, 202)))
model.add(
SimpleRNN(100, return_sequences=True, input_shape=(35, 202), unroll=True))
model.add(Dense(1000, input_dim=100, name='fc1'))
model.add(Activation('softmax'))
BATCH_SIZE = 128
NUM_EPOCHS = 50
learning_rate = 0.001
optim = adam(lr=learning_rate)
model.compile(loss="categorical_crossentropy",
optimizer=optim,
metrics=['accuracy'])
model.summary()
model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS)
def build_encoder_decoder_model(self,
image_encoder_latent_dim,
sentence_encoder_latent_dim,
words_to_idx,
word_embedding_size,
num_tokens,
num_stacked,
encoder_input_shape,
decoder_input_shape,
cell_type,
sentence_encoder,
masking=False,
recurrent_dropout=0.0,
input_dropout=0.0,
include_sentence_encoder=False,
attention=False):
if not include_sentence_encoder:
sentence_encoder_latent_dim = 0
# Shape (num_samples, 4096), 4096 is the image embedding length
encoder_inputs = Input(shape=encoder_input_shape,
name="encoder_input_layer")
# Masking layer, makes the network ignore 0 vectors
if masking:
mask_layer = Masking(mask_value=0, name="mask_layer")
mask_output = mask_layer(encoder_inputs)
# Defining encoder layer
encoder_lstm_name = "encoder_layer_"
for i in range(0, num_stacked):
if i == num_stacked - 1:
encoder = cell_type(image_encoder_latent_dim,
return_state=True,
recurrent_dropout=recurrent_dropout,
name=encoder_lstm_name + str(i))
else:
if i == 0:
input_dropout = input_dropout
else:
input_dropout = 0.0
encoder = cell_type(image_encoder_latent_dim,
return_sequences=True,
return_state=True,
recurrent_dropout=recurrent_dropout,
dropout=input_dropout,
name=encoder_lstm_name + str(i))
if i == 0:
if masking:
encoder_outputs = encoder(mask_output)
else:
encoder_outputs = encoder(encoder_inputs)
else:
encoder_outputs = encoder(encoder_outputs[0])
encoder_states = encoder_outputs[1:]
# Defining sentence encoder
if include_sentence_encoder:
# We can use the same inpt shape as the decoder
encoder_sentence_inputs = Input(
shape=decoder_input_shape, name="sentence_encoder_input_layer")
# Embedding layer that we don't train
sentence_encoder_embedding_layer = self.get_embedding_layer(
words_to_idx,
word_embedding_size,
num_tokens,
mask_zero=False,
name='sentence_embedding_layer')
sentence_embedding_outputs = sentence_encoder_embedding_layer(
encoder_sentence_inputs)
if attention:
sentence_encoder_outputs, _ = sentence_encoder.get_last_layer(
sentence_embedding_outputs, attention=True)
initial_encoder_states = encoder_states
else:
initial_encoder_states = sentence_encoder.get_last_layer(
sentence_embedding_outputs, encoder_states)
else:
initial_encoder_states = encoder_states
# Decoder input, should be shape (num_samples, 22)
decoder_inputs = Input(shape=decoder_input_shape,
name="decoder_input_layer")
# Embedding layer that we don't train
embedding_layer = self.get_embedding_layer(
words_to_idx,
word_embedding_size,
num_tokens,
mask_zero=True,
name='decoder_embedding_layer')
embedding_outputs = embedding_layer(decoder_inputs)
# Defining decoder layer
decoder_lstm_name = "decoder_layer_"
if attention:
decoder_latent_dim = sentence_encoder_latent_dim
else:
decoder_latent_dim = image_encoder_latent_dim + sentence_encoder_latent_dim
decoder_output = embedding_outputs
for i in range(0, num_stacked):
if attention and i == num_stacked - 1:
decoder = AttentionGRU(decoder_latent_dim,
return_sequences=True,
return_state=True,
name=decoder_lstm_name + str(i))
if i == 0:
decoder_outputs = decoder(
decoder_output,
initial_state=initial_encoder_states,
constants=sentence_encoder_outputs)
else:
decoder_outputs = decoder(
decoder_output, constants=sentence_encoder_outputs)
else:
decoder = cell_type(decoder_latent_dim,
return_sequences=True,
return_state=True,
recurrent_dropout=recurrent_dropout,
name=decoder_lstm_name + str(i))
if i == 0:
decoder_outputs = decoder(
decoder_output, initial_state=initial_encoder_states)
else:
decoder_outputs = decoder(decoder_output)
decoder_output = decoder_outputs[0]
dropout_layer = Dropout(input_dropout)
dropout_outputs = dropout_layer(decoder_outputs[0])
decoder_dense = TimeDistributed(Dense(num_tokens,
activation='softmax'),
name="dense_layer")
decoder_outputs = decoder_dense(dropout_outputs)
if include_sentence_encoder:
model = Model(
[encoder_inputs, encoder_sentence_inputs, decoder_inputs],
decoder_outputs)
else:
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
hash_index_input = Input(shape=(step_length, ))
encoder_embedding = Embedding(hash_vocab + 2,
hash_length,
weights=[hash_embedding],
mask_zero=True,
input_length=step_length)(hash_index_input)
pos_input = Input(shape=(step_length, pos_length))
chunk_input = Input(shape=(step_length, chunk_length))
gazetteer_input = Input(shape=(step_length, gazetteer_length))
senna_hash_pos_chunk_gazetteer_merge = merge(
[embedding, encoder_embedding, pos_input, chunk_input, gazetteer_input],
mode='concat')
input_mask = Masking(mask_value=0)(senna_hash_pos_chunk_gazetteer_merge)
hidden_1 = Bidirectional(LSTM(128, return_sequences=True))(input_mask)
hidden_2 = Bidirectional(LSTM(64, return_sequences=True))(hidden_1)
output = TimeDistributed(Dense(output_length, activation='softmax'))(hidden_2)
model = Model(input=[
embed_index_input, hash_index_input, pos_input, chunk_input,
gazetteer_input
],
output=output)
rmsprop = RMSprop(lr=0.0005)
model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['accuracy'])
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.models import load_model
from keras.callbacks import ModelCheckpoint
class LossHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
model = Sequential()
model.add(Masking(mask_value=0, input_shape=(xMaxLen, gloveSize)))
model.add(Bidirectional(LSTM(128, dropout_W=0.2, dropout_U=0.2)))
model.add(Dense(relationCnt, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
checkpointer = ModelCheckpoint(
filepath="model/relationDetection." + modelType + ".model",
verbose=1,
save_best_only=True,
)
history = LossHistory()
result = model.fit(X[:75708],
Y[:75708],
batch_size=batchSize,
nb_epoch=epoch,
def weight_average(inputs):
x = inputs[0]
y = inputs[1]
return (x+y)/2
def data_normal(x):
min_max_scaler = preprocessing.MinMaxScaler()
x = min_max_scaler.fit_transform(x)
return x
# Audio branch
# Previous 3793, 513, 98
frame_input = Input(shape=(535, 64))
mask_frame_input = Masking(mask_value=0.)(frame_input)
print('mask_frame_input shape: ', mask_frame_input.shape)
frame_l1 = Bidirectional(LSTM(100, return_sequences=True, recurrent_dropout=0.25, name='LSTM_audio_1'))(mask_frame_input)
frame_l1 = BatchNormalization()(frame_l1)
print('frame_l1 shape: ', frame_l1.shape)
frame_weight = AttentionLayer()(frame_l1)
frame_weight = BatchNormalization()(frame_weight)
print('frame_att shape: ', frame_weight.shape)
frame_weight_exp = Lambda(weight_expand)(frame_weight)
frame_att = Lambda(weight_dot)([frame_l1, frame_weight_exp])
frame_att = Lambda(lambda x: backend.sum(x, axis=1))(frame_att)
print('frame_att shape: ', frame_att.shape)
dropout_frame = Dropout(0.25)(frame_att)
model_frame = Model(frame_input, dropout_frame)
word_input = Input(shape=(98, 535, 64))
