def train(self, x, y, x_val, y_val, x_ts, y_ts):
early_stop = EarlyStopping(min_delta=0.01, patience=2)
save_path = self.name
save_best = ModelCheckpoint(save_path, save_best_only=True)
if self.use_multi_task:
self.train_model.fit(
x, [y, y, y, y],
validation_data=[x_val, [y_val, y_val, y_val, y_val]],
batch_size=128,
epochs=20,
callbacks=[early_stop, save_best])
else:
self.model.fit(x,
y,
validation_data=[x_val, y_val],
batch_size=128,
epochs=20,
callbacks=[early_stop, save_best])
metric = self.model.evaluate(x_ts, y_ts)
print(metric)
self.load_weights()
metric = self.model.evaluate(x_ts, y_ts, batch_size=512)
print(metric)
y_pred = self.model.predict(x_ts, batch_size=512)
cnf_matrix = confusion_matrix(convert_y(y_ts), convert_y(y_pred))
print(cnf_matrix)
def error_analysis(self, x_ts, y_ts, texts, start_index):
labels = ['人类作者', '自动摘要', '机器作者', '机器翻译']
y_pred = self.model.predict(x_ts, batch_size=512)
y_ts, y_pred = convert_y(y_ts), convert_y(y_pred)
with open('error.txt', 'w') as fout:
for i in range(y_ts.shape[0]):
if y_ts[i] != y_pred[i]:
fout.write('*****\n{}\n正确标签：{} 分类标签：{}\n'.format(
texts[start_index + i], labels[y_ts[i]],
labels[y_pred[i]]))
print('output error done.')
def test(self, x, ids, out_file):
labels = ['人类作者', '自动摘要', '机器作者', '机器翻译']
y_pred = self.model.predict(x, batch_size=512)
y_pred = convert_y(y_pred)
with open(out_file, 'w', encoding='utf-8') as fout:
for id, yi in zip(ids, y_pred):
label = labels[yi]
fout.write('{},{}\n'.format(id, label))
print('done.')
def fit(self, X_train, y_train):
self.X_train = X_train
self.y_train = y_train
skf = StratifiedKFold(n_splits=self.n_fold, shuffle=True, random_state=0)
# i = 0
y_test_list = []
y_pred_list = []
# for train_idx, test_idx in skf.split(self.X_train, self.y_train):
# X_train_s, y_train_s = self.X_train[train_idx], self.y_train[train_idx]
# X_test_s, y_test_s = self.X_train[test_idx], self.y_train[test_idx]
for i in range(self.n_fold):
print('cross ', i+1, '/', self.n_fold)
X_train_s, y_train_s, X_test_s, y_test_s = training_utils.split_cv(X_train, y_train, cv_num=self.n_fold, cv_index=i)
# X_train_s, X_val_s, y_train_s, y_val_s = train_test_split(X_train_s, y_train_s, test_size=0.1,
# stratify=y_train_s)
X_train_s, y_train_s, X_val_s, y_val_s = training_utils.split(X_train_s, y_train_s, split_ratio=0.95)
y_pred_s = None
for models in self.base_models:
model = models[i]
print(model.name)
if self.is_condition:
model.train_exp(X_train_s, y_train_s, X_val_s, y_val_s, X_test_s, y_test_s)
else:
model.train(X_train_s, y_train_s, X_val_s, y_val_s, X_test_s, y_test_s)
#作为特征的时候使用one-hot表示
if y_pred_s is None:
y_pred_s = model.predict(X_test_s)
else:
# import ipdb
# ipdb.set_trace( )
y_pred_s = np.hstack( (y_pred_s, model.predict(X_test_s) ) )
i += 1
y_test_list.append(y_test_s)
y_pred_list.append(y_pred_s)
# 使用y_pred_list做特征，y_test_list做目标，再次训练模型
X_top = np.vstack(y_pred_list)
y_top = np.vstack(y_test_list)
y_top = training_utils.convert_y( y_top )
if self.is_condition:
X_top = np.squeeze(X_top, axis=-1)
print(X_top.shape, y_top.shape)
self.top_model = LogisticRegression()
self.top_model.fit(X_top, y_top)
print(X_top)
def fit_tmp(self, X_train, y_train):
self.X_train = X_train
self.y_train = y_train
y_test_list = []
y_pred_list = []
for i in range(self.n_fold):
print('cross ', i+1, '/', self.n_fold)
X_train_s, y_train_s, X_test_s, y_test_s = training_utils.split_cv(X_train, y_train, cv_num=self.n_fold, cv_index=i)
X_train_s, y_train_s, X_val_s, y_val_s = training_utils.split(X_train_s, y_train_s, split_ratio=0.9)
y_pred_s = None
for models in self.base_models:
model = models[i]
print(model.name)
# if self.is_condition:
# model.train_exp(X_train_s, y_train_s, X_val_s, y_val_s, X_test_s, y_test_s)
# else:
# model.train(X_train_s, y_train_s, X_val_s, y_val_s, X_test_s, y_test_s)
#作为特征的时候使用one-hot表示
if y_pred_s is None:
y_pred_s = model.predict(X_test_s)
else:
# import ipdb
# ipdb.set_trace( )
y_pred_s = np.hstack( (y_pred_s, model.predict(X_test_s) ) )
i += 1
y_test_list.append(y_test_s)
y_pred_list.append(y_pred_s)
# 使用y_pred_list做特征，y_test_list做目标，再次训练模型
X_top = np.vstack(y_pred_list)
if self.is_condition:
X_top = np.squeeze(X_top, axis=-1)
y_top = np.vstack(y_test_list)
y_top = training_utils.convert_y( y_top )
print(X_top.shape, y_top.shape)
self.top_model = LogisticRegression()
self.top_model.fit(X_top, y_top)
print(X_top)
joblib.dump(self.top_model, self.name)
def predict():
"""
根据概率集成
:return:
"""
print('load data')
tn_conf = TrainConfigure()
data_dict = data_utils.pickle_load(tn_conf.char_file)
y = to_categorical(data_dict['y'])
x = data_dict['x']
xterm = data_utils.pickle_load(tn_conf.term_file)
xfeat = data_utils.pickle_load(tn_conf.feat_file)
# normalization
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(xfeat)
data_utils.pickle_dump(scaler, tn_conf.feat_norm)
xfeat = scaler.transform(xfeat)
print('loading embed ...')
term_vocab_dict = data_utils.pickle_load(tn_conf.term_dict)
term_embed_matrix = data_utils.load_embedding(
term_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/term_embed.pkl')
# term_embed_matrix = data_utils.load_embedding(term_vocab_dict,
# 'data/sgns.target.word-word.dynwin5.thr10.neg5.dim300.iter5',
# dump_path='data/term_embed_ww.pkl')
char_vocab_dict = data_utils.pickle_load(tn_conf.char_dict)
char_embed_matrix = data_utils.load_embedding(
char_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/char_embed.pkl')
print('load embed done.')
val_conf = ValidConfigure()
data_dict = data_utils.pickle_load(val_conf.char_file)
y = to_categorical(data_dict['y'])
x = data_dict['x']
ids = data_dict['id']
xterm = data_utils.pickle_load(val_conf.term_file)
xfeat = data_utils.pickle_load(val_conf.feat_file)
xfeat = scaler.transform(xfeat)
print('feat shape', xfeat.shape)
xtopic = data_utils.pickle_load('data/lda_vec_val.pkl')
xe = [[i for i in range(600)] for _ in range(y.shape[0])]
xe = np.array(xe)
xe_term = [[i for i in range(300)] for _ in range(y.shape[0])]
xe_term = np.array(xe_term)
import data_utils100
val_conf100 = data_utils100.ValidConfigure()
data_dict100 = data_utils.pickle_load(val_conf100.char_file)
x100 = data_dict100['x']
xterm100 = data_utils.pickle_load(val_conf100.term_file)
xe100 = [[i for i in range(100)] for _ in range(y.shape[0])]
xe100 = np.array(xe100)
xe_term100 = [[i for i in range(100)] for _ in range(y.shape[0])]
xe_term100 = np.array(xe_term100)
import data_utils200
val_conf200 = data_utils200.ValidConfigure()
data_dict200 = data_utils.pickle_load(val_conf200.char_file)
x200 = data_dict200['x']
xterm200 = data_utils.pickle_load(val_conf200.term_file)
xe200 = [[i for i in range(200)] for _ in range(y.shape[0])]
xe200 = np.array(xe200)
xe_term200 = [[i for i in range(200)] for _ in range(y.shape[0])]
xe_term200 = np.array(xe_term200)
ys = []
print('define model')
model = HybridDenseModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybriddensemodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
model = HybridDenseMAModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybriddensemodelma_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('dense model done.')
model = HybridSEModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridsemodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('se model done.')
# print('start len 100 model')
# model = HybridConvModel(char_embed_matrix=char_embed_matrix,
# term_embed_matrix=term_embed_matrix, MAX_LEN=100, MAX_LEN_TERM=100,NUM_FEAT=8,
# PE=True, name='hybridconvmodel_n100.h5')
# model.load_weights()
# y = model.predict([x100, xe100, xterm100, xe_term100, xfeat, xtopic])
# ys.append(y)
# del model
# print('hybrid conv model done.')
#
# model = HybridGatedDeepCNNModel(char_embed_matrix=char_embed_matrix,
# term_embed_matrix=term_embed_matrix, MAX_LEN=100, MAX_LEN_TERM=100,NUM_FEAT=8,
# PE=True, name='hybridgateddeepcnnmodel_n100.h5')
# model.load_weights()
# y = model.predict([x100, xe100, xterm100, xe_term100, xfeat, xtopic])
# ys.append(y)
# del model
# print('hybrid gated deep cnn model done.')
#
# model = HybridRCNNModel(char_embed_matrix=char_embed_matrix,
# term_embed_matrix=term_embed_matrix, MAX_LEN=100, MAX_LEN_TERM=100,NUM_FEAT=8,
# PE=True, name='hybridrcnnmodel_n100.h5')
# model.load_weights()
# y = model.predict([x100, xe100, xterm100, xe_term100, xfeat, xtopic])
# ys.append(y)
# del model
# print('hybrid RCNN model done.')
print('start len 200 model')
model = HybridConvModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
MAX_LEN=200,
MAX_LEN_TERM=200,
NUM_FEAT=8,
PE=True,
name='hybridconvmodel_n200.h5')
model.load_weights()
y = model.predict([x200, xe200, xterm200, xe_term200, xfeat, xtopic])
ys.append(y)
del model
print('hybrid conv model done.')
model = HybridDPCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
MAX_LEN=200,
MAX_LEN_TERM=200,
NUM_FEAT=8,
PE=True,
name='hybriddpcnnmodel_n200.h5')
model.load_weights()
y = model.predict([x200, xe200, xterm200, xe_term200, xfeat, xtopic])
ys.append(y)
del model
print('hybrid dpcnn model done.')
model = HybridGatedConvTopicModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
MAX_LEN=200,
MAX_LEN_TERM=200,
NUM_FEAT=8,
PE=True,
name='hybridgatedconvtopicmodel_n200.h5')
model.load_weights()
y = model.predict([x200, xe200, xterm200, xe_term200, xfeat, xtopic])
ys.append(y)
del model
print('hybrid dpcnn model done.')
model = HybridGatedDeepCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
MAX_LEN=200,
MAX_LEN_TERM=200,
NUM_FEAT=8,
PE=True,
name='hybridgateddeepcnnmodel_n200.h5')
model.load_weights()
y = model.predict([x200, xe200, xterm200, xe_term200, xfeat, xtopic])
ys.append(y)
del model
print('hybrid gated deep cnn model done.')
model = HybridRCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
MAX_LEN=200,
MAX_LEN_TERM=200,
NUM_FEAT=8,
PE=True,
name='hybridrcnnmodel_n200.h5')
model.load_weights()
y = model.predict([x200, xe200, xterm200, xe_term200, xfeat, xtopic])
ys.append(y)
del model
#这个模型太慢
model = ConditionAttModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='conditionattmodel_PE.h5',
lr=0.001)
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
print('condition att model done.')
model = ConditionConvModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='conditionconvmodel_PE.h5',
lr=0.001)
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('condition conv model done.')
model = ConditionDPCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='conditiondpcnnmodel_PE.h5',
lr=0.001)
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('condition dpcnn model done.')
model = ConditionGatedConvModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='conditiongatedconvmodel_PE.h5',
lr=0.001)
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('condition gated conv model done.')
model = ConditionGatedDeepCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='conditiongateddeepcnnmodel_PE.h5',
lr=0.001)
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('condition gated deepcnn model done.')
model = HybridAttModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridattmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
print('hybrid att model done.')
model = HybridConvModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridconvmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('hybrid conv model done.')
model = HybridDPCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybriddpcnnmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('hybrid dpcnn model done.')
model = HybridGatedDeepCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridgateddeepcnnmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('hybrid gated deep cnn model done.')
model = HybridRCNNModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridrcnnmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
del model
print('hybrid rcnn model done.')
model = HybridGatedConvTopicModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8,
PE=True,
name='hybridgatedconvtopicmodel_PE.h5')
model.load_weights()
y = model.predict([x, xe, xterm, xe_term, xfeat, xtopic])
ys.append(y)
print('hybrid gated conv topic done.')
y = fasttextmodel.predict_char()
ys.append(y)
y = fasttextmodel.predict_term()
ys.append(y)
print(y.shape)
print('fast text done.')
#hybrid model
# model = HybridModel(char_embed_matrix=char_embed_matrix, term_embed_matrix=term_embed_matrix, NUM_FEAT=8)# + 37
# model.load_weights()
# y = model.predict([x, xterm, xfeat])
# ys.append( y )
# print(y.shape)
# print('hybrid model done.')
labels = ['人类作者', '自动摘要', '机器作者', '机器翻译']
y_pred = np.mean(ys, axis=0)
y_pred = convert_y(y_pred)
out_file = 'result.csv'
with open(out_file, 'w', encoding='utf-8') as fout:
for id, yi in zip(ids, y_pred):
label = labels[yi]
fout.write('{},{}\n'.format(id, label))
print('done.')
def predict2():
"""
根据分类结果集成
:return:
"""
print('load data')
tn_conf = TrainConfigure()
data_dict = data_utils.pickle_load(tn_conf.char_file)
y = to_categorical(data_dict['y'])
x = data_dict['x']
xterm = data_utils.pickle_load(tn_conf.term_file)
xfeat = data_utils.pickle_load(tn_conf.feat_file)
# normalization
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(xfeat)
data_utils.pickle_dump(scaler, tn_conf.feat_norm)
xfeat = scaler.transform(xfeat)
print('loading embed ...')
term_vocab_dict = data_utils.pickle_load(tn_conf.term_dict)
term_embed_matrix = data_utils.load_embedding(
term_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/term_embed.pkl')
# term_embed_matrix = data_utils.load_embedding(term_vocab_dict,
# 'data/sgns.target.word-word.dynwin5.thr10.neg5.dim300.iter5',
# dump_path='data/term_embed_ww.pkl')
char_vocab_dict = data_utils.pickle_load(tn_conf.char_dict)
char_embed_matrix = data_utils.load_embedding(
char_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/char_embed.pkl')
print('load embed done.')
val_conf = ValidConfigure()
data_dict = data_utils.pickle_load(val_conf.char_file)
y = to_categorical(data_dict['y'])
x = data_dict['x']
ids = data_dict['id']
xterm = data_utils.pickle_load(val_conf.term_file)
xfeat = data_utils.pickle_load(val_conf.feat_file)
xfeat = scaler.transform(xfeat)
xe = [[i for i in range(600)] for _ in range(y.shape[0])]
xe = np.array(xe)
ys = []
print('define model')
#hybrid model
model = HybridModel(char_embed_matrix=char_embed_matrix,
term_embed_matrix=term_embed_matrix,
NUM_FEAT=8) # + 37
print('feat shape', xfeat.shape)
model.load_weights()
y = model.predict([x, xterm, xfeat])
ys.append(convert_onehot(y))
print('hybrid model done.')
#CNN model (char)
model = CharModel(embed_matrix=char_embed_matrix)
model.load_weights()
y = model.predict(x)
ys.append(convert_onehot(y))
print('char model done.')
model = CharModel(embed_matrix=char_embed_matrix,
name='charmodel_PE.h5',
PE=True)
model.load_weights()
y = model.predict([x, xe])
ys.append(convert_onehot(y))
print('char model done.')
model = CharModel(embed_matrix=char_embed_matrix,
name='charmodel_PE_OE.h5',
PE=True)
model.load_weights()
y = model.predict([x, xe])
ys.append(convert_onehot(y))
print('char model done.')
#CNN (term)
model = TermModel(embed_matrix=term_embed_matrix)
model.load_weights()
y = model.predict(xterm)
ys.append(convert_onehot(y))
print('term model done.')
model = DeepCNNModel(embed_matrix=char_embed_matrix)
model.load_weights()
y = model.predict(x)
ys.append(convert_onehot(y))
print('deep cnn done.')
# # attention model (char)
# model = AttModel(MAX_LEN=600, name='charattmodel.h5', embed_matrix=char_embed_matrix)
# model.load_weights()
# y = model.predict(x)
# ys.append(convert_onehot(y))
# # attention model (term)
# model = AttModel(MAX_LEN=300, embed_matrix=term_embed_matrix)
# model.load_weights()
# y = model.predict(xterm)
# ys.append(convert_onehot(y))
#
# model = ConditionModel(embed_matrix=char_embed_matrix)
# model.load_weights()
# y = model.predict(x)
# ys.append(convert_onehot(y))
model = SSCharModel(embed_matrix=char_embed_matrix,
name='sscharmodel_PE.h5',
PE=True,
train_embed=True)
model.load_weights()
y = model.predict([x, xe])
ys.append(convert_onehot(y))
model = SSCharModel(embed_matrix=char_embed_matrix, train_embed=True)
model.load_weights()
y = model.predict(x)
ys.append(convert_onehot(y))
model = GatedConvModel(embed_matrix=char_embed_matrix,
name='gatedconvmodel_PE.h5',
PE=True)
model.load_weights()
y = model.predict([x, xe])
ys.append(convert_onehot(y))
model = GatedConvModel(embed_matrix=char_embed_matrix, train_embed=True)
model.load_weights()
y = model.predict(x)
ys.append(convert_onehot(y))
model = GatedDeepCNNModel(embed_matrix=char_embed_matrix,
name='gateddeepcnnmodel_PE.h5',
PE=True,
train_embed=True)
model.load_weights()
y = model.predict([x, xe])
ys.append(convert_onehot(y))
model = GatedDeepCNNModel(embed_matrix=char_embed_matrix, train_embed=True)
model.load_weights()
y = model.predict(x)
ys.append(convert_onehot(y))
labels = ['人类作者', '自动摘要', '机器作者', '机器翻译']
y_pred = np.mean(ys, axis=0)
y_pred = convert_y(y_pred)
out_file = 'result.csv'
with open(out_file, 'w', encoding='utf-8') as fout:
for id, yi in zip(ids, y_pred):
label = labels[yi]
fout.write('{},{}\n'.format(id, label))
print('done.')
def stacking_main_condition():
print('load data')
tn_conf = TrainConfigure()
data_dict = data_utils.pickle_load(tn_conf.char_file)
y = to_categorical(data_dict['y'])
x = data_dict['x']
xterm = data_utils.pickle_load(tn_conf.term_file)
xfeat = data_utils.pickle_load(tn_conf.feat_file)
# normalization
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(xfeat)
data_utils.pickle_dump(scaler, tn_conf.feat_norm)
xfeat = scaler.transform(xfeat)
xe = [[i for i in range(600)] for _ in range(y.shape[0])]
xe = np.array(xe)
xe_term = [[i for i in range(300)] for _ in range(y.shape[0])]
xe_term = np.array(xe_term)
xtopic = data_utils.pickle_load('data/lda_vec.pkl')
print('loading embed ...')
term_vocab_dict = data_utils.pickle_load(tn_conf.term_dict)
term_embed_matrix = data_utils.load_embedding(term_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/term_embed.pkl')
# term_embed_matrix = data_utils.load_embedding(term_vocab_dict,
# 'data/sgns.target.word-word.dynwin5.thr10.neg5.dim300.iter5',
# dump_path='data/term_embed_ww.pkl')
char_vocab_dict = data_utils.pickle_load(tn_conf.char_dict)
char_embed_matrix = data_utils.load_embedding(char_vocab_dict,
'data/sgns.target.word-character.char1-2.dynwin5.thr10.neg5.dim300.iter5',
dump_path='data/char_embed.pkl')
print('load embed done.')
name = 'model/stack_condition_model.pkl'
model_dir = 'model/stack/'
n_fold = 3
name = 'model/stack_condition_model5.pkl'
model_dir = 'model/stack5/'
n_fold = 5
stk_model = stacking(n_fold, name=name, is_condition=True)
conf = conditionmodelbase.ModelConfigure()
conf.PE = True
stk_model.add_model(ConditionConvModel, {"conf":conf,"char_embed_matrix":char_embed_matrix,
"term_embed_matrix":term_embed_matrix,
"name":model_dir+'conditionconvmodel_PE.h5'})
stk_model.add_model(ConditionGatedConvModel, {"conf":conf,"char_embed_matrix": char_embed_matrix,
"term_embed_matrix": term_embed_matrix,
"name": model_dir+'conditiongatedconvmodel_PE.h5'})
stk_model.add_model(ConditionGatedDeepCNNModel, {"conf":conf,"char_embed_matrix": char_embed_matrix,
"term_embed_matrix": term_embed_matrix,
"name": model_dir+'conditiongateddeepcnnmodel_PE.h5'})
conf.lr = 0.0005
stk_model.add_model(ConditionDPCNNModel, {"conf": conf, "char_embed_matrix": char_embed_matrix,
"term_embed_matrix": term_embed_matrix,
"name": model_dir + 'conditiondpcnnmodel_PE.h5'})
#采样0.1用于测试
# x_tn, y_tn, x_ts, y_ts = training_utils.split([x, xe, xterm, xe_term, xfeat, xtopic], y, split_ratio=0.005, shuffle=False)
# x_tn, y_tn, x_ts, y_ts = training_utils.split(x_tn, y_tn, shuffle=False)
x_tn, y_tn, x_ts, y_ts = training_utils.split([x, xe, xterm, xe_term, xfeat, xtopic], y, split_ratio=0.95)
stk_model.fit(x_tn, y_tn)
# joblib.dump(stk_model, 'model/stack_model_3.pkl')
y_pred = stk_model.predict(x_ts)
acc = accuracy_score(training_utils.convert_y(y_pred), training_utils.convert_y(y_ts) )
print(acc)
cnf_matrix = confusion_matrix(training_utils.convert_y(y_pred), training_utils.convert_y(y_ts) )
print(cnf_matrix)
stk_model.save( )