def train():
word_to_id, id_to_word = load_vocab(args.vocab_file)
tag_to_id, id_to_tag = load_vocab(args.tag_file)
processing_word = get_processing_word(word_to_id)
processing_tag = get_processing_word(tag_to_id, allow_unk=False)
# load data sets
train_sentences = LoadDataset(args.train_file, processing_word,
processing_tag)
dev_sentences = LoadDataset(args.dev_file, processing_word, processing_tag)
test_sentences = LoadDataset(args.test_file, processing_word,
processing_tag)
# Use selected tagging scheme (IOB / IOBES)
# update_tag_scheme(train_sentences, args.tag_schema)
# update_tag_scheme(test_sentences, args.tag_schema)
if os.path.isfile(args.config_file):
config = load_config(args.config_file)
else:
config = config_model(word_to_id, tag_to_id, id_to_tag)
save_config(config, args.config_file)
make_path(args)
log_path = os.path.join("log", args.log_file)
logger = get_logger(log_path)
with tf.Session() as sess:
model = create_model(sess, Model, args.ckpt_path, load_word2vec,
config, logger)
model.train(train_sentences, dev_sentences)
def set_log_dir(self, new_dir):
self.log_dir = new_dir
if not os.path.exists(self.log_dir):
os.makedirs(self.log_dir)
self.out_dir = os.path.abspath(
os.path.join(self.log_dir, str(self.version)))
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
self.saved_model_dir = os.path.abspath(
os.path.join(self.out_dir, "checkpoints"))
self.checkpoint_prefix = os.path.join(self.saved_model_dir, "model")
self.train_summary_dir = os.path.join(self.out_dir, "summaries",
"train")
self.dev_summary_dir = os.path.join(self.out_dir, "summaries", "dev")
self.logger = get_logger(os.path.join(self.out_dir, 'log'))
def evaluate_line():
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
while True:
# try:
# line = input("请输入测试句子:")
# result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag)
# print(result)
# except Exception as e:
# logger.info(e)
line = input("请输入测试句子:")
result = model.evaluate_line(sess,
input_from_line(line, char_to_id),
id_to_tag)
print(result)
from model.text import Vocab
from torch.nn.parallel import DistributedDataParallel
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--config', help='config file', default='config.json')
parser.add_argument('--gpu', help='which gpu to use', type=str, default='4')
parser.add_argument("--local_rank",
help='used for distributed training',
type=int,
default=-1)
args = parser.parse_args()
config = utils.load_config(args.config)
config_path = os.path.dirname(args.config)
logger = utils.get_logger(os.path.join(config_path, 'main.log'))
train_dir = os.path.join(config_path, config['train_dir'])
data_dir = os.path.join(config_path, config['data_dir'])
eval_dir = os.path.join(config_path, config['eval_dir'])
log_dir = os.path.join(config_path, config['log_dir'])
best_model = os.path.join(config_path, config['best_dir'])
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# helpers -----------------------------------------------------
def save_func(epoch, device):
filename = utils.get_ckpt_filename('model', epoch)
torch.save(trainer.state_dict(), os.path.join(train_dir, filename))
if os.path.exists(
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data # the length of batch data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True,
batch) # step是什么意思？
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# use dev data to validation the model
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
# use current the best model to test
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
@Desc :
'''
import os
import logging
import jieba
import jieba.posseg as pseg
from collections import deque
import pathlib
from model.utils import (get_logger, similarity)
jieba.dt.tmp_dir = "./"
jieba.default_logger.setLevel(logging.ERROR)
logger = get_logger('faqrobot', logfile="faqrobot.log")
basedir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent)
faqdata = basedir + '/data/FAQ.txt'
class zhishiku():
def __init__(self, q):
self.q = [q]
self.a = ""
self.sim = 0
self.q_vec = []
self.q_word = []
def __str__(self):