def print_config_summary(config, args):
# Summarize config on the screen as a sanity check
print(100 * "=")
print(f"Here is the configuration being used:\n")
print_config(config)
print(100 * "=")
print(f"Arguments being used:\n")
print_config(args)
print(100 * "=")
def main_train():
# load data sets
# sentences = [[(words11, tag11), ...], [(word21, tag21), ...], ...]
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
# 更新在train_sentences和test_sentences中
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
# 创建或加载字符、词、特征、target的映射字典
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
# 若存在pre-trained embedding file，则同时使用pre-trained和训练集构建字典
if FLAGS.pre_emb:
# 统计train中字，返回字典
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
# 使用pre-trained的字增大训练集字的字典
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
# 否则只是用训练集构建字典
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, target_to_id, id_to_target = tag_mapping(train_sentences)
# 创建其他特征的映射字典，返回的三个都为dict
_f, feature_to_id, id_to_feature = feature_mapping(train_sentences, FLAGS.features)
# 存储字、target、feature的映射关系
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, target_to_id, id_to_target, feature_to_id, id_to_feature], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, target_to_id, id_to_target, feature_to_id, id_to_feature = pickle.load(f)
# make path for store log and model config if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = create_config_model(FLAGS, char_to_id, target_to_id, feature_to_id)
logger = get_logger(FLAGS.log_file)
print_config(config, logger)
train(config, train_sentences, dev_sentences, test_sentences, char_to_id, feature_to_id, target_to_id, id_to_char,
id_to_target, logger)
parser.add_argument('--batch_size', type=int, default=3, help='Batch size')
parser.add_argument('--gpu_device', type=str, default=0, help='GPU device (number) to use; defaults to 0')
parser.add_argument('--cpu', action='store_true', help='Whether to use CPU instead of GPU; this option overwrites the --gpu_device argument')
parser.add_argument('--save_scores', action='store_true', help='Whether to save model scores during inference')
parser.add_argument('--split', default='test', help='Dataset split; possible values are \'training\', \'validation\', \'test\'')
args = parser.parse_args()
with open(os.path.join(args.config_path, 'config.pkl'), 'rb') as f:
cfg_dict = pickle.load(f)
if 'merge_function' not in cfg_dict:
cfg_dict['merge_function'] = 'concat'
if 'zero_shot' not in cfg_dict:
cfg_dict['zero_shot'] = False
cfg_dict['classes_to_exclude'] = None
if 'num_features_mixed5c' not in cfg_dict:
cfg_dict['num_features_mixed5c'] = 1024
#if 'i3d_weights_path' not in cfg_dict:
# cfg_dict['i3d_weights_path'] = 'models/'
cfg_dict['i3d_weights_path'] = 'models/'
if 'class_map' not in cfg_dict:
cfg_dict['class_map'] = utils.class2idx_map(cfg_dict['classes_to_exclude'])
cfg = config.GetConfig(**cfg_dict)
utils.print_config(cfg)
dataloader, test_set, model, device = prepare_inference(cfg, args)
inference(dataloader, test_set, model, device, cfg)
import matplotlib.pyplot as plt
from collections import OrderedDict
import h5py
from utils.utils import prepare_output_dirs, print_config, write_config
from utils.feature_extraction import read_data
from utils.model import create_model
from utils.visdata import save_history, plot_confusion_matrix
from utils.SGDW import SGDW
from config import parse_opts
config = parse_opts()
config = prepare_output_dirs(config)
print_config(config)
write_config(config, os.path.join(config.save_dir, 'config.json'))
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = config.device
####################################################################
####################################################################
# Parameters
song_samples = 660000
genres = {
'metal': 0,
'disco': 1,
'classical': 2,
'hiphop': 3,
def train_ner():
clean(FLAGS)
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(25):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train():
# load data sets
train_sentences = load_sentences(args.train_file, args.lower, args.zeros)
dev_sentences = load_sentences(args.dev_file, args.lower, args.zeros)
test_sentences = load_sentences(args.test_file, args.lower, args.zeros)
# Use selected tagging scheme (IOB / IOBES)
# 检测并维护数据集的 tag 标记
update_tag_scheme(train_sentences, args.tag_schema)
update_tag_scheme(test_sentences, args.tag_schema)
update_tag_scheme(dev_sentences, args.tag_schema)
# create maps if not exist
# 根据数据集创建 char_to_id, id_to_char, tag_to_id, id_to_tag 字典，并储存为 pkl 文件
if not os.path.isfile(args.map_file):
# create dictionary for word
if args.pre_emb:
dico_chars_train = char_mapping(train_sentences, args.lower)[0]
# 利用预训练嵌入集增强（扩充）字符字典，然后返回字符与位置映射关系
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), args.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
args.lower)
# Create a dictionary and a mapping for tags
# 获取标记与位置映射关系
tag_to_id, id_to_tag, intent_to_id, id_to_intent = tag_mapping(
train_sentences)
with open(args.map_file, "wb") as f:
pickle.dump([
char_to_id, id_to_char, tag_to_id, id_to_tag, intent_to_id,
id_to_intent
], f)
else:
with open(args.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag, intent_to_id, id_to_intent = pickle.load(
f)
# 提取句子特征
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
intent_to_id, args.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
intent_to_id, args.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
intent_to_id, args.lower)
# code.interact(local=locals())
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
# 获取可供模型训练的单个批次数据
train_manager = BatchManager(train_data, args.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(args)
if os.path.isfile(args.config_file):
config = load_config(args.config_file)
else:
config = config_model(char_to_id, tag_to_id, intent_to_id)
save_config(config, args.config_file)
make_path(args)
logger = get_logger(args.log_file)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
# 训练集全量跑一次需要迭代的次数
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
# 此处模型创建为项目最核心代码
model = create_model(sess, Model, args.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss_slot = []
loss_intent = []
# with tf.device("/gpu:0"):
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss_slot, batch_loss_intent = model.run_step(
sess, True, batch)
loss_slot.append(batch_loss_slot)
loss_intent.append(batch_loss_intent)
if step % args.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"INTENT loss:{:>9.6f}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss_intent),
np.mean(loss_slot)))
loss_slot = []
loss_intent = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
# if i%7 == 0:
save_model(sess, model, args.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def prepare_training(cfg, args):
if cfg.use_gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.device_list
if cfg.use_gpu and torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
if cfg.model_name == 'baseline': # initalize baseline model
model = baseline_model.BaseNet(cfg)
elif cfg.model_name == 'gcn': # initalize gcn model
model = gcn_model.GCNNet(cfg)
else:
assert (
False
), 'Variable model_name should be either \'baseline\' or 2 \'gcn\'.'
model = model.to(device=device)
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=1,
momentum=cfg.momentum,
weight_decay=cfg.weight_decay)
if cfg.resume_training:
cfg.filename = 'checkpoint_' + cfg.checkpoint_path.split('/')[-2]
model, optimizer, start_epoch = load_checkpoint(
model, optimizer, cfg.checkpoint_path)
cfg.start_epoch = start_epoch
else:
cfg.filename = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime())
utils.make_dirs(
os.path.join(cfg.results_path, cfg.model_name, cfg.filename))
utils.save_config(cfg)
utils.print_config(cfg)
if cfg.save_scores:
utils.make_dirs(
os.path.join(cfg.scores_path, cfg.model_name, cfg.filename,
datasets['train'].split))
utils.make_dirs(
os.path.join(cfg.scores_path, cfg.model_name, cfg.filename,
datasets['val'].split))
if cfg.plot_grad_flow:
utils.make_dirs(
os.path.join(cfg.results_path, cfg.model_name, cfg.filename,
'grad_flow'))
#utils.save_config(cfg)
#utils.print_config(cfg)
training_set = dataset.return_dataset(args.data_path, args.annot_path, cfg,
'training')
validation_set = dataset.return_dataset(args.data_path, args.annot_path,
cfg, 'validation')
datasets = {'train': training_set, 'val': validation_set}
training_loader = data.DataLoader(training_set,
batch_size=cfg.training_batch_size,
shuffle=True,
num_workers=4,
worker_init_fn=worker_init_fn)
validation_loader = data.DataLoader(validation_set,
batch_size=cfg.validation_batch_size,
shuffle=False,
num_workers=4)
dataloaders = {'train': training_loader, 'val': validation_loader}
return dataloaders, datasets, model, device, optimizer
def run_gdl(cfg: DictConfig) -> None:
"""
Function general for Geo Deep-Learning using Hydra library to rules all the
parameters and functions using during the task wanted to execute.
Process
-------
1. Read and convert the `gdl_config.yaml` to a dictionary.
2. Verify if the code and other information need to be save.
3. If the debug option is activate, the entire config yaml will be printed
and save in a log. In addition of that, if the mode is `train`, a
validation run will precede the training to assure the well functioning
of the code.
4. Verify is the chosen mode is available.
5. Verify is the chosen task is available and run the code link to that
task.
-------
:param cfg: (DictConfig) Parameters and functions in the main yaml config
file.
"""
cfg = OmegaConf.create(cfg)
# debug config
if cfg.debug:
pass
# cfg.training.num_sanity_val_steps = 1 # only work with pytorch lightning
# logging.info(OmegaConf.to_yaml(cfg, resolve=True))
# check if the mode is chosen
if type(cfg.mode) is DictConfig:
msg = "You need to choose between those modes: {}"
raise logging.critical(msg.format(list(cfg.mode.keys())))
# save all overwritten parameters
logging.info('\nOverwritten parameters in the config: \n' +
cfg.general.config_override_dirname)
# Start -----------------------------------
msg = "Let's start {} for {} !!!".format(cfg.mode, cfg.task.task_name)
logging.info("\n" + "-" * len(msg) + "\n" + msg + "\n" + "-" * len(msg))
# -----------------------------------------
# Start the timer
start_time = time.time()
# Read the task and execute it
task = load_obj(cfg.task.path_task_function)
task(cfg)
# Add git hash from current commit to parameters.
with open_dict(cfg):
cfg.general.git_hash = get_git_hash()
# Pretty print config using Rich library
if cfg.get("print_config"):
print_config(cfg, resolve=True)
# End --------------------------------
msg = "End of {} !!!".format(cfg.mode)
logging.info("\n" + "-" * len(msg) + "\n" + msg + "\n" +
"Elapsed time: {:.2f}s".format(time.time() - start_time) +
"\n" + "-" * len(msg) + "\n")
def train():
tf.io.gfile.mkdir(FLAGS.output)
log_path = os.path.join(FLAGS.output, 'model.log')
logger = get_logger(log_path)
# load data sets
train_sentences = load_sentences(os.path.join(FLAGS.data, "train.txt"),
FLAGS.zeros)
dev_sentences = load_sentences(os.path.join(FLAGS.data, "dev.txt"),
FLAGS.zeros)
test_sentences = load_sentences(os.path.join(FLAGS.data, "test.txt"),
FLAGS.zeros)
# create maps if not exist
map_file = os.path.join(FLAGS.output, 'maps.pkl')
if not os.path.isfile(map_file):
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(map_file, "wb") as f:
pickle.dump([tag_to_id, id_to_tag], f)
else:
with open(map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, FLAGS.max_seq_len, tag_to_id)
dev_data = prepare_dataset(dev_sentences, FLAGS.max_seq_len, tag_to_id)
test_data = prepare_dataset(test_sentences, FLAGS.max_seq_len, tag_to_id)
logger.info("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, FLAGS.batch_size)
test_manager = BatchManager(test_data, FLAGS.batch_size)
# make path for store log and model if not exist
config_file = os.path.join(FLAGS.output, 'config.json')
if os.path.isfile(config_file):
config = load_config(config_file)
else:
config = config_model(tag_to_id)
save_config(config, config_file)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model,
os.path.join(FLAGS.output, 'checkpoint'), config,
logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess,
model,
os.path.join(FLAGS.output, 'checkpoint'),
logger,
global_steps=step)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
#update_tag_scheme(train_sentences, FLAGS.tag_schema)
#update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
os.makedirs('%s' % FLAGS.save_path)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_padding_dataset(train_sentences, FLAGS.max_seq_len,
char_to_id, tag_to_id, FLAGS.lower)
dev_data = prepare_padding_dataset(dev_sentences, FLAGS.max_seq_len,
char_to_id, tag_to_id, FLAGS.lower)
test_data = prepare_padding_dataset(test_sentences, FLAGS.max_seq_len,
char_to_id, tag_to_id, FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
"""
batch = train_manager.batch_data[0]
strings, chars, segs, tags = batch
for chrs in chars:
print(chrs)
for chrs in segs:
print(chrs)
print(tag_to_id)
"""
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join(FLAGS.save_path, "log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = TransformerCRFModel(config, is_training=True)
sess.run(tf.global_variables_initializer())
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
predict_lists = []
source_tag = []
best_dev_f1 = 0.0
best_test_f1 = 0.0
for batch in dev_manager.iter_batch(shuffle=False):
lengths, logits = model.run_step(sess, False, batch)
_, chars, segs, tags = batch
transition = model.transition.eval(session=sess)
pre_seq = model.predict(logits, transition, lengths)
pre_label = recover_label(pre_seq, lengths, id_to_tag)
"""
for p in range(len(pre_label)):
print(chars[p])
print(pre_label[p])
"""
source_label = recover_label(tags, lengths, id_to_tag)
predict_lists.extend(pre_label)
source_tag.extend(source_label)
train_loss_v = np.round(float(np.mean(loss)), 4)
print('****************************************************')
acc, p, r, f = get_ner_fmeasure(source_tag, predict_lists,
config["tag_schema"])
logger.info('epoch:\t{}\ttrain loss:\t{}\t'.format(
i + 1, train_loss_v))
logger.info('dev acc:\t{}\tp:\t{}\tr:\t{}\tf:\t{}'.format(
acc, p, r, f))
for batch in test_manager.iter_batch(shuffle=False):
lengths, logits = model.run_step(sess, False, batch)
_, chars, segs, tags = batch
transition = model.transition.eval(session=sess)
pre_seq = model.predict(logits, transition, lengths)
pre_label = recover_label(pre_seq, lengths, id_to_tag)
source_label = recover_label(tags, lengths, id_to_tag)
predict_lists.extend(pre_label)
source_tag.extend(source_label)
acc_t, p_t, r_t, f_t = get_ner_fmeasure(source_tag, predict_lists,
config["tag_schema"])
logger.info('test acc:\t{}\tp:\t{}\tr:\t{}\tf:\t{}'.format(
acc_t, p_t, r_t, f_t))
if f > best_dev_f1:
save_model(sess, model, FLAGS.ckpt_path, logger)
best_dev_f1 = f
best_test_f1 = f_t
logger.info(
'save epoch:\t{} model with best dev f1-score'.format(i +
1))
print('****************************************************\n\n')