def train(text_version='ek', label_version=None, config_path='c93ftri.yaml'):
"""
python -m algo.main93_v2 train
python3 -m algo.main93_v2 train -c config_ntua93.yaml
:param text_version: string
:param label_version: string
:param config_path: string
:return:
"""
config_data = yaml.load(open(config_path))
output_key = 'f_{}_{}_{}'.format(NNModel.name, text_version,
int(time.time()))
if label_version is not None:
output_key = '{}_{}'.format(label_version, output_key)
print('OUTPUT_KEY: {}'.format(output_key))
# 准备输出路径的文件夹
data_config.prepare_output_folder(output_key=output_key)
data_config.prepare_model_folder(output_key=output_key)
shutil.copy(config_path, data_config.output_path(output_key, ALL, CONFIG))
w2v_key = '{}_{}'.format(config_data['word']['w2v_version'], text_version)
w2v_model_path = data_config.path(ALL, WORD2VEC, w2v_key)
vocab_train_path = data_config.path(TRAIN, VOCAB, text_version)
# 加载字典集
# 在模型中会采用所有模型中支持的词向量, 并为有足够出现次数的单词随机生成词向量
vocab_meta_list = load_vocab_list(vocab_train_path)
vocabs = [
_meta['t'] for _meta in vocab_meta_list
if _meta['tf'] >= config_data['word']['min_tf']
]
# 加载词向量与相关数据
lookup_table, vocab_id_mapping, embedding_dim = load_lookup_table2(
w2v_model_path=w2v_model_path, vocabs=vocabs)
json.dump(
vocab_id_mapping,
open(data_config.output_path(output_key, ALL, VOCAB_ID_MAPPING), 'w'))
# 加载配置
nn_config = NNConfig(config_data)
train_config = TrainConfig(config_data['train'])
early_stop_metric = train_config.early_stop_metric
# 加载训练数据
datasets = dict()
datasets[TRAIN], output_dim = load_dataset(
mode=[TRAIN, TEST],
vocab_id_mapping=vocab_id_mapping,
max_seq_len=nn_config.seq_len,
label_version=label_version,
sampling=train_config.train_sampling,
filter_others=True)
# 初始化数据集的检索
index_iterators = {
TRAIN: IndexIterator.from_dataset(datasets[TRAIN]),
}
# 按配置将训练数据切割成训练集和验证集
index_iterators[TRAIN].split_train_valid(train_config.valid_rate)
# 计算各个类的权重
if train_config.use_class_weights:
label_weight = {
# 参考 sklearn 中 class_weight='balanced'的公式, 实验显示效果显着
_label: float(index_iterators[TRAIN].n_sample()) /
(index_iterators[TRAIN].dim * len(_index))
for _label, _index in index_iterators[TRAIN].label_index.items()
}
else:
label_weight = {
_label: 1.
for _label in range(index_iterators[TRAIN].dim)
}
# 基于加载的数据更新配置
nn_config.set_embedding_dim(embedding_dim)
nn_config.set_output_dim(output_dim)
# 搭建神经网络
nn = NNModel(config=nn_config)
nn.build_neural_network(lookup_table=lookup_table)
batch_size = train_config.batch_size
fetches = {
mode: {_key: nn.var(_key)
for _key in fetch_key[mode]}
for mode in [TRAIN, TEST]
}
model_output_prefix = data_config.model_path(key=output_key) + '/model'
best_res = {mode: None for mode in [TRAIN, VALID]}
no_update_count = {mode: 0 for mode in [TRAIN, VALID]}
max_no_update_count = 10
eval_history = {TRAIN: list(), VALID: list(), TEST: list()}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
dataset = datasets[TRAIN]
index_iterator = index_iterators[TRAIN]
# 训练开始 ##########################################################################
for epoch in range(train_config.epoch):
print('== epoch {} = {} ='.format(epoch, output_key))
# 利用训练集进行训练
print('TRAIN')
n_sample = index_iterator.n_sample(TRAIN)
labels_predict = list()
labels_gold = list()
for batch_index in index_iterator.iterate(batch_size,
mode=TRAIN,
shuffle=True):
feed_dict = {
nn.var(_key): dataset[_key][batch_index]
for _key in feed_key[TRAIN]
}
feed_dict[nn.var(SAMPLE_WEIGHTS)] = list(
map(label_weight.get, feed_dict[nn.var(LABEL_GOLD)]))
feed_dict[nn.var(TEST_MODE)] = 0
res = sess.run(fetches=fetches[TRAIN], feed_dict=feed_dict)
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += dataset[LABEL_GOLD][batch_index].tolist()
labels_predict, labels_gold = labels_predict[:
n_sample], labels_gold[:
n_sample]
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
print_evaluation(res)
eval_history[TRAIN].append(res)
global_step = tf.train.global_step(sess, nn.var(GLOBAL_STEP))
if train_config.valid_rate == 0.:
if best_res[TRAIN] is None or res[
early_stop_metric] > best_res[TRAIN][early_stop_metric]:
best_res[TRAIN] = res
no_update_count[TRAIN] = 0
saver.save(sess,
save_path=model_output_prefix,
global_step=global_step)
else:
no_update_count[TRAIN] += 1
else:
if best_res[TRAIN] is None or res[
early_stop_metric] > best_res[TRAIN][early_stop_metric]:
best_res[TRAIN] = res
no_update_count[TRAIN] = 0
else:
no_update_count[TRAIN] += 1
# 计算在验证集上的表现, 不更新模型参数
print('VALID')
n_sample = index_iterator.n_sample(VALID)
labels_predict = list()
labels_gold = list()
for batch_index in index_iterator.iterate(batch_size,
mode=VALID,
shuffle=False):
feed_dict = {
nn.var(_key): dataset[_key][batch_index]
for _key in feed_key[TEST]
}
feed_dict[nn.var(TEST_MODE)] = 1
res = sess.run(fetches=fetches[TEST], feed_dict=feed_dict)
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += dataset[LABEL_GOLD][batch_index].tolist()
labels_predict, labels_gold = labels_predict[:
n_sample], labels_gold[:
n_sample]
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
eval_history[VALID].append(res)
print_evaluation(res)
# Early Stop
if best_res[VALID] is None or res[
early_stop_metric] > best_res[VALID][early_stop_metric]:
saver.save(sess,
save_path=model_output_prefix,
global_step=global_step)
best_res[VALID] = res
no_update_count[VALID] = 0
else:
no_update_count[VALID] += 1
if no_update_count[TRAIN] >= max_no_update_count:
break
# 训练结束 ##########################################################################
# 确保输出文件夹存在
print(
'========================= BEST ROUND EVALUATION ========================='
)
json.dump(eval_history,
open(data_config.output_path(output_key, 'eval', 'json'), 'w'))
labels_predict_ = dict()
labels_gold_ = dict()
with tf.Session() as sess:
prefix_checkpoint = tf.train.latest_checkpoint(
data_config.model_path(key=output_key))
saver = tf.train.import_meta_graph('{}.meta'.format(prefix_checkpoint))
saver.restore(sess, prefix_checkpoint)
nn = BaseNNModel(config=None)
nn.set_graph(tf.get_default_graph())
for mode in [TRAIN, TEST, FINAL]:
dataset, _ = load_dataset(mode=mode,
vocab_id_mapping=vocab_id_mapping,
max_seq_len=nn_config.seq_len,
label_version=label_version)
index_iterator = SimpleIndexIterator.from_dataset(dataset)
n_sample = index_iterator.n_sample()
prob_predict = list()
labels_predict = list()
labels_gold = list()
for batch_index in index_iterator.iterate(batch_size,
shuffle=False):
feed_dict = {
nn.var(_key): dataset[_key][batch_index]
for _key in feed_key[TEST]
}
feed_dict[nn.var(TEST_MODE)] = 1
res = sess.run(fetches=fetches[TEST], feed_dict=feed_dict)
prob_predict += res[PROB_PREDICT].tolist()
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += dataset[LABEL_GOLD][batch_index].tolist()
prob_predict = prob_predict[:n_sample]
labels_predict = labels_predict[:n_sample]
labels_gold = labels_gold[:n_sample]
labels_predict_[mode] = labels_predict
labels_gold_[mode] = labels_gold
# 导出预测的label
with open(data_config.output_path(output_key, mode, LABEL_PREDICT),
'w') as file_obj:
for _label in labels_predict:
file_obj.write('{}\n'.format(_label))
with open(data_config.output_path(output_key, mode, PROB_PREDICT),
'w') as file_obj:
for _prob in prob_predict:
file_obj.write('\t'.join(map(str, _prob)) + '\n')
print('VALID')
res = best_res[VALID]
print_evaluation(res)
for col in res[CONFUSION_MATRIX]:
print(','.join(map(str, col)))
print()
print('TRAIN + TEST')
gold = labels_gold_[TRAIN] + labels_gold_[TEST]
pred = labels_predict_[TRAIN] + labels_predict_[TEST]
select_index = build_select_index(gold)
res = basic_evaluate(gold=filter_by_index(gold, select_index),
pred=filter_by_index(pred, select_index))
print_evaluation(res)
for col in res[CONFUSION_MATRIX]:
print(','.join(map(str, col)))
print()
print('FINAL')
gold = labels_gold_[FINAL]
pred = labels_predict_[FINAL]
select_index = build_select_index(gold)
res = basic_evaluate(gold=filter_by_index(gold, select_index),
pred=filter_by_index(pred, select_index))
print_evaluation(res)
for col in res[CONFUSION_MATRIX]:
print(','.join(map(str, col)))
print()
print('OUTPUT_KEY: {}'.format(output_key))
def train(text_version='ek',
label_version=None,
config_path='config93_naive.yaml'):
"""
python -m algo.main93_v2 train
python3 -m algo.main93_v2 train -c config_ntua93.yaml
:param text_version: string
:param label_version: string
:param config_path: string
:return:
"""
config_data = yaml.load(open(config_path))
output_key = '{}_{}_{}'.format(NNModel.name, text_version,
int(time.time()))
if label_version is not None:
output_key = '{}_{}'.format(label_version, output_key)
print('OUTPUT_KEY: {}'.format(output_key))
# 准备输出路径的文件夹
data_config.prepare_output_folder(output_key=output_key)
data_config.prepare_model_folder(output_key=output_key)
shutil.copy(config_path, data_config.output_path(output_key, ALL, CONFIG))
w2v_key = '{}_{}'.format(config_data['word']['w2v_version'], text_version)
w2v_model_path = data_config.path(ALL, WORD2VEC, w2v_key)
vocab_train_path = data_config.path(TRAIN, VOCAB, text_version)
# 加载字典集
# 在模型中会采用所有模型中支持的词向量, 并为有足够出现次数的单词随机生成词向量
vocab_meta_list = load_vocab_list(vocab_train_path)
vocabs = [
_meta['t'] for _meta in vocab_meta_list
if _meta['tf'] >= config_data['word']['min_tf']
]
# 加载词向量与相关数据
lookup_table, vocab_id_mapping, embedding_dim = load_lookup_table2(
w2v_model_path=w2v_model_path, vocabs=vocabs)
json.dump(
vocab_id_mapping,
open(data_config.output_path(output_key, ALL, VOCAB_ID_MAPPING), 'w'))
# 加载配置
nn_config = NNConfig(config_data)
train_config = TrainConfig(config_data['train'])
early_stop_metric = train_config.early_stop_metric
# 加载训练数据
datasets, output_dim = load_dataset(vocab_id_mapping=vocab_id_mapping,
max_seq_len=nn_config.seq_len,
with_label=True,
label_version=label_version)
datasets['all_train'] = datasets[TRAIN]
datasets[TRAIN], datasets[VALID] = split_train_valid(
dataset=datasets['all_train'], valid_rate=train_config.valid_rate)
index_iterators = {
mode: SimpleIndexIterator.from_dataset(datasets[mode])
for mode in [VALID, TEST, 'all_train']
}
for mode in [VALID, TEST, 'all_train']:
datasets[mode] = dataset_as_input(datasets[mode], nn_config.seq_len)
label_weight = {_label: 1. for _label in range(output_dim)}
# 基于加载的数据更新配置
nn_config.set_embedding_dim(embedding_dim)
nn_config.set_output_dim(output_dim)
# 搭建神经网络
nn = NNModel(config=nn_config)
nn.build_neural_network(lookup_table=lookup_table)
batch_size = train_config.batch_size
fetches = {
mode: {_key: nn.var(_key)
for _key in fetch_key[mode]}
for mode in [TRAIN, TEST]
}
model_output_prefix = data_config.model_path(key=output_key) + '/model'
best_res = {mode: None for mode in [TRAIN, VALID]}
no_update_count = {mode: 0 for mode in [TRAIN, VALID]}
max_no_update_count = 10
eval_history = {TRAIN: list(), VALID: list(), TEST: list()}
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
# 训练开始 ##########################################################################
for epoch in range(train_config.epoch):
print('== epoch {} = {} =='.format(epoch, output_key))
# 利用训练集进行训练
print('TRAIN')
dataset = custom_sampling(datasets[TRAIN])
index_iterator = SimpleIndexIterator.from_dataset(dataset=dataset)
dataset = dataset_as_input(dataset, nn_config.seq_len)
n_sample = index_iterator.n_sample()
labels_predict = list()
labels_gold = list()
for batch_index in index_iterator.iterate(batch_size,
shuffle=True):
feed_dict = {
nn.var(_key): dataset[_key][batch_index]
for _key in feed_key[TRAIN]
}
feed_dict[nn.var(SAMPLE_WEIGHTS)] = list(
map(label_weight.get, feed_dict[nn.var(LABEL_GOLD)]))
feed_dict[nn.var(TEST_MODE)] = 0
res = sess.run(fetches=fetches[TRAIN], feed_dict=feed_dict)
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += dataset[LABEL_GOLD][batch_index].tolist()
labels_predict, labels_gold = labels_predict[:
n_sample], labels_gold[:
n_sample]
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
print_evaluation(res)
eval_history[TRAIN].append(res)
global_step = tf.train.global_step(sess, nn.var(GLOBAL_STEP))
if train_config.valid_rate == 0.:
if best_res[TRAIN] is None or res[
early_stop_metric] > best_res[TRAIN][early_stop_metric]:
best_res[TRAIN] = res
no_update_count[TRAIN] = 0
saver.save(sess,
save_path=model_output_prefix,
global_step=global_step)
else:
no_update_count[TRAIN] += 1
else:
if best_res[TRAIN] is None or res[
early_stop_metric] > best_res[TRAIN][early_stop_metric]:
best_res[TRAIN] = res
no_update_count[TRAIN] = 0
else:
no_update_count[TRAIN] += 1
# 计算在验证集上的表现, 不更新模型参数
print('VALID')
_mode = VALID
_dataset = datasets[_mode]
_index_iterator = SimpleIndexIterator.from_dataset(_dataset)
_n_sample = _index_iterator.n_sample()
labels_predict = list()
labels_gold = list()
for batch_index in _index_iterator.iterate(batch_size,
shuffle=False):
feed_dict = {
nn.var(_key): _dataset[_key][batch_index]
for _key in feed_key[TEST]
}
feed_dict[nn.var(TEST_MODE)] = 1
res = sess.run(fetches=fetches[TEST], feed_dict=feed_dict)
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += _dataset[LABEL_GOLD][batch_index].tolist()
labels_predict, labels_gold = labels_predict[:
_n_sample], labels_gold[:
_n_sample]
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
eval_history[_mode].append(res)
print_evaluation(res)
# Early Stop
if best_res[VALID] is None or res[
early_stop_metric] > best_res[VALID][early_stop_metric]:
saver.save(sess,
save_path=model_output_prefix,
global_step=global_step)
best_res[VALID] = res
no_update_count[VALID] = 0
else:
no_update_count[VALID] += 1
# eval test
_mode = TEST
_dataset = datasets[_mode]
_index_iterator = SimpleIndexIterator.from_dataset(_dataset)
_n_sample = _index_iterator.n_sample()
labels_predict = list()
labels_gold = list()
for batch_index in _index_iterator.iterate(batch_size,
shuffle=False):
feed_dict = {
nn.var(_key): _dataset[_key][batch_index]
for _key in feed_key[TEST]
}
feed_dict[nn.var(TEST_MODE)] = 1
res = sess.run(fetches=fetches[TEST], feed_dict=feed_dict)
labels_predict += res[LABEL_PREDICT].tolist()
labels_gold += _dataset[LABEL_GOLD][batch_index].tolist()
labels_predict, labels_gold = labels_predict[:
_n_sample], labels_gold[:
_n_sample]
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
eval_history[_mode].append(res)
print('TEST')
print_evaluation(res)
if no_update_count[TRAIN] >= max_no_update_count:
break
# 训练结束 ##########################################################################
# 确保输出文件夹存在
print(
'========================= BEST ROUND EVALUATION ========================='
)
json.dump(eval_history,
open(data_config.output_path(output_key, 'eval', 'json'), 'w'))
with tf.Session() as sess:
prefix_checkpoint = tf.train.latest_checkpoint(
data_config.model_path(key=output_key))
saver = tf.train.import_meta_graph('{}.meta'.format(prefix_checkpoint))
saver.restore(sess, prefix_checkpoint)
nn = BaseNNModel(config=None)
nn.set_graph(tf.get_default_graph())
for mode in [TRAIN, TEST]:
dataset = datasets[mode if mode == TEST else 'all_train']
index_iterator = SimpleIndexIterator.from_dataset(dataset)
n_sample = index_iterator.n_sample()
prob_predict = list()
labels_predict = list()
labels_gold = list()
hidden_feats = list()
for batch_index in index_iterator.iterate(batch_size,
shuffle=False):
feed_dict = {
nn.var(_key): dataset[_key][batch_index]
for _key in feed_key[TEST]
}
feed_dict[nn.var(TEST_MODE)] = 1
res = sess.run(fetches=fetches[TEST], feed_dict=feed_dict)
prob_predict += res[PROB_PREDICT].tolist()
labels_predict += res[LABEL_PREDICT].tolist()
hidden_feats += res[HIDDEN_FEAT].tolist()
labels_gold += dataset[LABEL_GOLD][batch_index].tolist()
prob_predict = prob_predict[:n_sample]
labels_predict = labels_predict[:n_sample]
labels_gold = labels_gold[:n_sample]
hidden_feats = hidden_feats[:n_sample]
if mode == TEST:
res = basic_evaluate(gold=labels_gold, pred=labels_predict)
best_res[TEST] = res
# 导出隐藏层
with open(data_config.output_path(output_key, mode, HIDDEN_FEAT),
'w') as file_obj:
for _feat in hidden_feats:
file_obj.write('\t'.join(map(str, _feat)) + '\n')
# 导出预测的label
with open(data_config.output_path(output_key, mode, LABEL_PREDICT),
'w') as file_obj:
for _label in labels_predict:
file_obj.write('{}\n'.format(_label))
with open(data_config.output_path(output_key, mode, PROB_PREDICT),
'w') as file_obj:
for _prob in prob_predict:
file_obj.write('\t'.join(map(str, _prob)) + '\n')
for mode in [TRAIN, VALID, TEST]:
res = best_res[mode]
print(mode)
print_evaluation(res)
print()
json.dump(best_res,
open(data_config.output_path(output_key, ALL, 'best_eval'), 'w'))
test_score_list = map(lambda _item: _item['f1'], eval_history[TEST])
print('best test f1 reached: {}'.format(max(test_score_list)))
print('OUTPUT_KEY: {}'.format(output_key))
