def evaluate_predictsamples():
config = load_config(FLAGS.config_file)
logger = get_logger('log/test.log')
print_config(config, logger)
# limit GPU memory
logger.info("start predict")
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
index = 1
with open('data/predict.txt', 'w', encoding='utf-8') as f_write:
read_file = 'data/test.txt'
for line in open(read_file, 'r', encoding='utf-8-sig').readlines():
result = model.evaluate_line(sess,
input_from_line(line, char_to_id),
id_to_tag)
print(result)
content = str(index) + ','
if result is not None:
entities = result['entities']
for entity in entities:
if entity:
print(entity['word'] + '\t' + entity['type'])
# content += entity['word'] + '\t' + str(entity['start']) + '\t' + str(entity['end']) + '\t' + entity['type']+ ';'
content += entity['word'] + '\t' + entity[
'type'] + ';'
f_write.write(content + '\n')
index += 1
def main():
# read arguments
args = parse_arguments()
args['device'] = torch.device('cuda')
#args['device'] = torch.device('cpu')
load_config(args)
print_config(args)
# load data
data = load_data(args)
train_data, test_data = data
args['spect_dim'] = train_data.size()[-1]
args['time_step'] = train_data.size()[-2]
# create model
#model_low, model_high, joint_model_low, joint_model_high = create_model(args)
#model_low = model_low.to(args['device'])
#model_high = model_high.to(args['device'])
#joint_model_low = joint_model_low.to(args['device'])
#joint_model_high = joint_model_high.to(args['device'])
model = create_model(args)
model = model.to(args['device'])
if args['action'] == 'train':
# initialize training latents and train model
train_latents, train_loss, best_loss, best_epoch, best_model = optimize_network(args, model, train_data, 'train')
model = best_model.to(args['device'])
test_loss, final_loss, final_pred, test_latents = optimize_network(args, model, test_data, 'test')
elif args['action'] == 'test':
# initialize testing latents and test model
test_loss, final_loss, final_pred, test_latents = optimize_network(args, model, test_data, 'test')
# save statistics
basedir = "/projects/dataset_processed/xiaoyam"
#basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
model_folder = os.path.join(basedir, 'model_saves', 'pytorch',
args['config_folder'], 'config_' + str(args['config_num']))
if not os.path.isdir(model_folder):
os.makedirs(model_folder)
save_results(args, model_folder, {
'train_latents': train_latents,
'test_latents': test_latents,
'test_loss': test_loss,
'final_test_loss': final_loss,
'train_loss': train_loss,
'final_pred': final_pred,
'best_train_loss': best_loss,
'best_train_epoch': best_epoch,
'best_model': model,
'spect_true': test_data,
})
def main():
# add argumentation
parser = argparse.ArgumentParser(
description='MobileNet_v2_DeepLab_v3 Pytorch Implementation')
parser.add_argument(
'--dataset',
default='cityscapes',
choices=['cityscapes', 'other'],
help='Dataset used in training MobileNet v2+DeepLab v3')
parser.add_argument('--root',
default='./data/cityscapes',
help='Path to your dataset')
parser.add_argument('--epoch',
default=None,
help='Total number of training epoch')
parser.add_argument('--lr', default=None, help='Base learning rate')
parser.add_argument('--pretrain',
default=None,
help='Path to a pre-trained backbone model')
parser.add_argument('--resume_from',
default=None,
help='Path to a checkpoint to resume model')
args = parser.parse_args()
params = Params()
# parse args
if not os.path.exists(args.root):
if params.dataset_root is None:
raise ValueError('ERROR: Root %s not exists!' % args.root)
else:
params.dataset_root = args.root
if args.epoch is not None:
params.num_epoch = args.epoch
if args.lr is not None:
params.base_lr = args.lr
if args.pretrain is not None:
params.pre_trained_from = args.pretrain
if args.resume_from is not None:
params.resume_from = args.resume_from
LOG('Network parameters:')
print_config(params)
# create dataset and transformation
LOG('Creating Dataset and Transformation......')
datasets = create_dataset(params)
LOG('Creation Succeed.\n')
# create model
LOG('Initializing MobileNet and DeepLab......')
net = MobileNetv2_DeepLabv3(params, datasets)
LOG('Model Built.\n')
# let's start to train!
net.Train()
net.Test()
def __init__(self,config):
self.config = config
print_config(config)
# build session
sess_config = tf.ConfigProto(device_count = {"CPU":self.config.num_cpu},
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=0)
self.sess = tf.Session(config=sess_config)
def do_config(self, arg):
"""View or change the configuration."""
# TODO
if not arg:
utils.print_config()
return
arg = self.split_args()
print(arg)
print("*** Not implemented yet.")
def main():
# read arguments
args = parse_arguments()
args['device'] = torch.device('cuda')
#args['device'] = torch.device('cpu')
load_config(args)
print_config(args)
# load data
data = load_data(args)
(train_spect, train_pcm), (test_spect, test_pcm) = data
args['spect_dim'] = train_spect.size()[-1]
args['pcm_dim'] = train_pcm.size()[-1]
# create model
model = create_model(args)
model = model.to(args['device'])
if args['action'] == 'train':
# initialize training latents and train model
train_latents, train_loss, best_loss, best_epoch, best_model = optimize_network(
args, model, train_spect, train_pcm, 'train')
model = best_model.to(args['device'])
test_loss, final_loss, final_pred, test_latents = optimize_network(
args, model, test_spect, test_pcm, 'test')
elif args['action'] == 'test':
# initialize testing latents and test model
test_loss, final_loss, final_pred, test_latents = optimize_network(
args, model, test_spect, test_pcm, 'test')
# save statistics
basedir = os.getcwd()
#basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
model_folder = os.path.join(basedir, 'model_saves', 'pytorch',
args['config_folder'],
'config_' + str(args['config_num']))
if not os.path.isdir(model_folder):
os.makedirs(model_folder)
save_results(
args, model_folder, {
'train_latents': train_latents,
'test_latents': test_latents,
'test_loss': test_loss,
'final_test_loss': final_loss,
'train_loss': train_loss,
'final_pred': final_pred,
'best_train_loss': best_loss,
'best_train_epoch': best_epoch,
'best_model': model,
'spect_true': test_spect
})
def main():
# add argumentation
parser = argparse.ArgumentParser(description='MobileNet_v2_DeepLab_v3 Pytorch Implementation')
#todo maybe make it work with multiple datasets?
#parser.add_argument('--dataset', default='cityscapes', choices=['cityscapes', 'other'],
# help='Dataset used in training MobileNet v2+DeepLab v3')
parser.add_argument('--root', default='./data/cityscapes', help='Path to your dataset')
parser.add_argument('--epoch', default=None, help='Total number of training epoch')
parser.add_argument('--lr', default=None, help='Base learning rate')
parser.add_argument('--pretrain', default=None, help='Path to a pre-trained backbone model')
parser.add_argument('--resume_from', default=None, help='Path to a checkpoint to resume model')
parser.add_argument('--logdir', default=None, help='Directory to save logs for Tensorboard')
parser.add_argument('--batch_size', default=128, help='Batch size for training')
args = parser.parse_args()
params = Params()
# parse args
if not os.path.exists(args.root):
if params.dataset_root is None:
raise ValueError('ERROR: Root %s doesn\'t exist!' % args.root)
else:
params.dataset_root = args.root
if args.epoch is not None:
params.num_epoch = int(args.epoch)
if args.lr is not None:
params.base_lr = args.lr
if args.pretrain is not None:
params.pre_trained_from = args.pretrain
if args.resume_from is not None:
params.resume_from = args.resume_from
if args.logdir is not None:
params.logdir = args.logdir
params.summary_dir, params.ckpt_dir = create_train_dir(params.logdir)
params.train_batch = int(args.batch_size)
LOG('Network parameters:')
print_config(params)
# create dataset and transformation
LOG('Creating Dataset and Transformation......')
datasets = create_dataset(params)
LOG('Creation Succeed.\n')
# create model
LOG('Initializing MobileNet and DeepLab......')
net = MobileNetv2_DeepLabv3(params, datasets)
LOG('Model Built.\n')
# let's start to train!
net.Train()
net.Test()
def train_model(model_module, config, params=None, values=None, do_evaluate=False):
if params is not None:
for param, value in zip(params, values):
assert param in config.__dict__
setattr(config, param, value)
print_config(config)
data_builder = model_module.DataBuilder(config)
data_builder.run()
config_holder = model_module.ConfigHolder(config)
model = model_module.Model(config_holder)
model.build()
train = model_module.CoNLLDataset(config_holder.filename_train,
config_holder.processing_word_train,
config_holder.processing_tag,
config_holder.max_iter,
use_buckets=config.bucket_train_data,
batch_size=config.batch_size,
shuffle=config.shuffle_train_data,
sort=config.sort_train_data)
test = model_module.CoNLLDataset(config_holder.filename_dev,
config_holder.processing_word_infer,
config_holder.processing_tag,
config_holder.max_iter,
sort=True)
train_eval = model_module.CoNLLDataset(config_holder.filename_train,
config_holder.processing_word_infer,
config_holder.processing_tag,
sort=True,
max_iter=config_holder.train_sentences_to_eval)
model.train(train, test, train_eval)
model.close_session()
tf.reset_default_graph()
# read accuracies for all iterations
df = pd.read_csv(config_holder.training_log,
names=['epoch', 'acc_train', 'acc_test', 'train_loss', 'nbatches',
'epoch_time', 'train_time', 'eval_time'])
acc_train_list = df['acc_train']
acc_test_list = df['acc_test']
train_loss_list = df['train_loss']
# evaluate
if do_evaluate is True:
print("Evaluating...")
evaluate(model, config_holder, config_holder.filename_dev, 'LANG_NA', 'dev', config_holder.out_dir)
model.close_session()
tf.reset_default_graph()
return acc_train_list, acc_test_list, train_loss_list
def get_config(self):
"""
从模型参数配置文件中获取参数或者用config_model函数生成参数并存储
:return:日志logger及参数列表config
"""
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = self.config_model(self.char_to_id, self.tag_to_id)
save_config(config, FLAGS.config_file)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
return logger, config
def main(args):
# print config
utils.print_config(args)
if args.config:
return
id_file_path = args.id_file
overwrite_ids = args.update_ids
dont_have_id_file = not os.path.exists(id_file_path)
data_file_path = args.output_file
update_data = args.update_data
dont_have_data = not os.path.exists(data_file_path)
download_attachments = args.download
add_to_ipfs = args.ipfs
add_to_qri = args.qri
#-------------------------------------------------------------------
if overwrite_ids or dont_have_id_file:
fetch_valid_id_list(args)
else:
print("...using existing id list '{}'".format(id_file_path))
#-------------------------------------------------------------------
if update_data or dont_have_data:
extract_data_from_pages(args)
else:
print("...using existing data file '{}'".format(data_file_path))
#-------------------------------------------------------------------
if download_attachments:
fetch_attachment_files(args)
else:
print("...skipping attachment download")
#-------------------------------------------------------------------
if add_to_ipfs:
add_to_ipfs_and_save_results(args)
else:
print("...skipping IPFS add")
#-------------------------------------------------------------------
if add_to_qri:
data_file_path = args.output_file.replace(".json", "_with_hashes.json")
utils.add_or_save_to_qri(args.dsname, data_file_path,
"templates/structure.json",
"templates/meta.json")
else:
print("...skipping QRI add")
def first_menu():
menu = -1
while menu == -1 or menu == 1:
try:
menu = int(
input("""
1. Print current config.
2. Config interfaces.
3. Choose competition phase (Firewall rules).
0. Quit.
"""))
if menu == 1:
print_config()
elif menu == 2:
change_config()
create_netplan_config_interactive(if_list)
elif menu == 3:
second_menu()
elif (menu) == 0:
sys.exit
else:
print("Your choice (%d) is wrong. Please, try again." % (menu))
except ValueError:
print('ERRORE di input.')
elif config['hyperparameters']['lr_multistep'] != 'None':
tmp_str = "lr_scheduler : [milestones: "
for milestone in scheduler_for_lr.milestones:
tmp_str = tmp_str + str(milestone) + ', '
tmp_str += ' gamma: '
tmp_str += str(scheduler_for_lr.gamma)
tmp_str += ']'
print(tmp_str)
print("Size of batch : " + str(train_loader.batch_size))
print("transform : " + str(transform))
print("num. train data : " + str(len(train_dataset)))
print("num. valid data : " + str(len(valid_dataset)))
print("num_classes : " + str(num_classes))
print("classes : " + str(encoder.dict))
utils.print_config(config)
input("Press any key to continue..")
def train(epoch):
crnn.train()
generator_g.train()
discriminator_ga.train()
discriminator_fa.train()
train_loss = 0
global global_iter_train
batch_size = train_loader.batch_size
with torch.set_grad_enabled(True):
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, char_to_id, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, char_to_id, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
#print batch
step, batch_loss = model.run_step(sess, True, batch)
#print step
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train():
# load data sets
datasets = load_sentences(FLAGS.train_file, FLAGS.lower)
random.shuffle(datasets)
train_sentences = datasets[:14000]
test_sentences = datasets[14000:]
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
char_to_id, _ = elmo_char_mapping(FLAGS.elmo_vocab)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i sentences in train / dev." %
(len(train_data), len(test_data)))
elmo_batcher = get_batcher()
train_manager = BatchManager(train_data, FLAGS.batch_size, elmo_batcher)
test_manager = BatchManager(test_data, FLAGS.batch_size, elmo_batcher)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto(allow_soft_placement=True)
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
elmo_model = load_elmo()
model = create_model(sess, Model, FLAGS.ckpt_path, elmo_model, config,
logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info(
"iteration:{} step:{}/{}, NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch, steps_per_epoch,
np.mean(loss)))
loss = []
best = evaluate(sess, model, "test", test_manager, id_to_tag,
logger)
# evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
#update_tag_scheme(train_sentences, FLAGS.tag_schema)
#update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), 0, len(test_data)))
train_len=len(train_data)
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, FLAGS.batch_size)
test_manager = BatchManager(test_data, FLAGS.batch_size)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
from tqdm import tqdm
for batch in tqdm(train_manager.iter_batch(shuffle=True)):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
print("save result epoch:",i," ***************************************************")
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger,i)
if i>=8:
save_model(sess, model, FLAGS.ckpt_path, logger, global_steps=step)
evaluate(sess, model, "test", test_manager, id_to_tag, logger,i)
def train():
# 加载数据集
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# 选择tag schema(IOB / IOBES) I：中间，O：其他，B：开始 | E：结束，S：单个
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file): # 配置文件：char_to_id, id_to_char, tag_to_id, id_to_tag的数据
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences, FLAGS.id_to_tag_path, FLAGS.tag_to_id_path)
# with open('maps.txt','w',encoding='utf8') as f1:
# f1.writelines(str(char_to_id)+" "+id_to_char+" "+str(tag_to_id)+" "+id_to_tag+'\n')
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) #
# prepare data, get a collection of list containing index
# train_data[0][0]：一句话；
# train_data[0][1]：单个字的编号；
# train_data[0][2]：切词之后，切词特征：词的大小是一个字的话是0，词的大小是2以上的话：1,2....，2,3；
# train_data[0][3]：每个字的标签
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, char_to_id, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, char_to_id, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size) # 按batch size将数据拆分
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger)
logger.info("start training")
loss = []
# tf.device("/cpu:0") 指定运行的GPU(默认为GPU:0)
with tf.device("/cpu:0"):
for i in range(100):
# 按批次训练模型。这个是训练的开始，可以从这里倒着找整个网络怎么训练
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
# 打印信息：
# iteration：迭代次数，也就是经过多少个epoch；
#
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
# best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if i % 7 == 0:
save_model(sess, model, FLAGS.ckpt_path, logger)
#长度不足补0
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# GPU设置
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.model_path, load_word2vec, config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
def main():
# load data sets
global args
args = parser.parse_args()
pp.pprint(vars(args))
# running_name = 'X'
use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu)
# use_cuda = False
# train_file = 'data/example.train'
# dev_file = 'data/example.dev'
test_file = 'data/example.test'
# embedding_file = 'data/vec.txt'
map_file = 'map.pkl'
# config_file = 'config_file_pytorch'
tag_file = 'tag.pkl'
# embedding_easy_file = 'data/easy_embedding.npy'
# train_sentences = load_sentences(train_file)
# dev_sentences = load_sentences(dev_file)
test_sentences = load_sentences(test_file)
# train_sentences = dev_sentences
# update_tag_scheme(train_sentences, args.tag_schema)
update_tag_scheme(test_sentences, args.tag_schema)
# update_tag_scheme(dev_sentences, args.tag_schema)
if not os.path.isfile(tag_file):
print("Tag file {:s} Not found".format(tag_file))
sys.exit(-1)
else:
with open(tag_file, 'rb') as t:
tag_to_id, id_to_tag = pickle.load(t)
if not os.path.isfile(map_file):
print("Map file {:s} Not found".format(map_file))
# create dictionary for word
# dico_chars_train = char_mapping(train_sentences)[0]
# dico_chars, char_to_id, id_to_char = augment_with_pretrained(
# dico_chars_train.copy(),
# embedding_file,
# list(itertools.chain.from_iterable(
# [[w[0] for w in s] for s in test_sentences])
# )
# )
# # _, tag_to_id, id_to_tag = tag_mapping(train_sentences)
#
# with open(map_file, "wb") as f:
# pickle.dump([char_to_id, id_to_char], f)
else:
with open(map_file, "rb") as f:
char_to_id, id_to_char = pickle.load(f)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id)
print("{:d} sentences in test.".format(len(test_data)))
test_manager = BatchManager(test_data, 1)
save_places = dir_utils.save_places(args.eval)
# log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(
os.path.join(save_places.log_save_dir,
'evaluation-{:d}.txt'.format(args.fileid)))
config = config_model(char_to_id, tag_to_id, args)
print_config(config, logger)
logger.info("start training")
#Update: create model and embedding!
model = NERModel.CNERPointer(char_dim=args.char_dim,
seg_dim=args.seg_dim,
hidden_dim=args.hidden_dim,
max_length=15,
output_classes=4,
dropout=args.dropout,
embedding_path=None,
id_to_word=id_to_char,
easy_load=None)
print("Number of Params\t{:d}".format(
sum([p.data.nelement() for p in model.parameters()])))
#Update: this won't work!
# model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu)
if use_cuda:
model = model.cuda()
model.eval()
if args.eval is not None:
# if os.path.isfile(args.resume):
ckpt_filename = os.path.join(
save_places.model_save_dir,
'checkpoint_{:04d}.pth.tar'.format(args.fileid))
assert os.path.isfile(
ckpt_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_filename,
map_location=lambda storage, loc: storage)
model.load_state_dict(checkpoint['state_dict'], strict=True)
train_iou = checkpoint['IoU']
print("=> loading checkpoint '{}', current iou: {:.04f}".format(
ckpt_filename, train_iou))
ner_results = evaluate(model, test_manager, id_to_tag, use_cuda, max_len=5)
eval_lines = test_ner(ner_results, save_places.summary_save_dir)
for line in eval_lines:
logger.info(line)
f1 = float(eval_lines[1].strip().split()[-1])
return f1
# Delay time in seconds for API call returns in case of auth
# failures (some DoS-countermeasure). For testing purposes we
# usually want this as fast as possible. This value gets used in
# create_mq_objects.py for the creation of the queue manager conn
# auth.
FAIL_DELAY = os.environ.get('PYMQI_TEST_QM_CONN_AUTH_FAIL_DELAY',
'0')
# Queue naming setup
class QUEUE:
PREFIX = os.environ.get('PYMQI_TEST_QUEUE_PREFIX', '')
QUEUE_NAMES = {
'TestRFH2PutGet': PREFIX + 'TEST.PYMQI.RFH2PUTGET',
'TestQueueManager': PREFIX + 'TEST.PYMQI.QUEUEMANAGER',
}
# convenience attribute derived from above settings, may be used for tests
# that mandate the MQSERVER environment variable
# E.g. MQSERVER="SVRCONN.1/TCP/mq.example.org(1777)"
MQSERVER = '%(channel)s/%(transport)s/%(host)s(%(port)s)' % {
'channel': QM.CHANNEL,
'transport': QM.TRANSPORT,
'host': QM.HOST,
'port': QM.PORT,
}
if __name__ == '__main__':
utils.print_config(PATHS, MQ)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
# "sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'I-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'I-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'I-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'I-TYPE']]"
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
# print("train_sentences[0]:{}".format(train_sentences[0]))
# "train_sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'E-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'E-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'E-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'E-TYPE']]"
update_tag_scheme(dev_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
# print("map_file:{}".format(FLAGS.map_file))
# print("pre_emb:{}".format(FLAGS.pre_emb))
# map_file: maps.pkl
# pre_emb: False
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(
train_sentences, FLAGS.lower)[0] # character -> count dict
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
# '3027 / 0 / 361 sentences in train / dev / test.'
# print("batch_size:{}".format(FLAGS.batch_size))
# batch_size: 20
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
# print("config_file:{}".format(FLAGS.config_file))
# config_file: config_file
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
log_path = os.path.join("log", FLAGS.log_file)
# log_path:log/train.log
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
# print("steps_per_epoch:{}".format(steps_per_epoch))
# steps_per_epoch: 152
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
# print("steps_check:{}".format(FLAGS.steps_check))
# steps_check: 100
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
export(model, sess, "ner", "export_model")
def train():
# load data sets
# 句子集合 = [[句子1],[句子2],[句子3]]，句子1 = [我 O，在 O，。。。]
#<class 'list'>: [['海', 'O'], ['钓', 'O'], ['比', 'O'], ['赛', 'O'], ['地', 'O'], ['点', 'O'], ['在', 'O'], ['厦', 'B-LOC'], ['门', 'I-LOC'], ['与', 'O'], ['金', 'B-LOC'], ['门', 'I-LOC'], ['之', 'O'], ['间', 'O'], ['的', 'O'], ['海', 'O'], ['域', 'O'], ['。', 'O']]
# train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
# dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
# test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
from xlnet_base.xlnet_data_utils import XLNetDataUtils
sp_model = spm.SentencePieceProcessor()
sp_model.Load('./chinese_xlnet_base_L-12_H-768_A-12/spiece.model')
train_data = XLNetDataUtils(sp_model,
batch_size=FLAGS.batch_size,
entry="train")
dev_data = XLNetDataUtils(sp_model,
batch_size=FLAGS.batch_size,
entry="dev")
test_data = XLNetDataUtils(sp_model,
batch_size=FLAGS.batch_size,
entry="test")
dev_batch = dev_data.iteration()
def datapadding(data):
alldatalist = []
datalist = data.data
max_length = 64
for i in range(len(datalist)):
tmpdatalist = []
token = datalist[i][0]
segmentid = datalist[i][1]
inputid = datalist[i][2]
inputmask = datalist[i][3]
labellist = datalist[i][4]
#token label
if len(labellist) < max_length:
for i in range(max_length - len(token)):
labellist.append(0)
elif len(labellist) > max_length:
tmplabellist = []
for i in range(max_length):
tmplabellist.append(labellist[i])
labellist = tmplabellist
#segmentid inputid inputmask
if len(segmentid) < max_length:
for i in range(max_length - len(segmentid)):
segmentid.append(0)
inputid.append(0)
inputmask.append(0)
elif len(segmentid) > max_length:
tmpsegmentid = []
tmpinputid = []
tmpinputmask = []
for i in range(max_length):
tmpsegmentid.append(segmentid[i])
tmpinputid.append(inputid[i])
tmpinputmask.append(inputmask[i])
segmentid = tmpsegmentid
inputid = tmpinputid
inputmask = tmpinputmask
tmpdatalist.append(token)
tmpdatalist.append(segmentid)
tmpdatalist.append(inputid)
tmpdatalist.append(inputmask)
tmpdatalist.append(labellist)
alldatalist.append(tmpdatalist)
return alldatalist
ftraindata = datapadding(train_data)
fdevdata = datapadding(dev_data)
ftestdata = datapadding(test_data)
print(len(ftraindata))
print(len(fdevdata))
print(len(ftestdata))
# traindata = {
# "batch_size": train_data.batch_size,
# "input_size": train_data.input_size,
# "vocab": train_data.vocab,
# "tag_map": train_data.tag_map,
# }
# devdata = {
# "batch_size": dev_data.batch_size,
# "input_size": dev_data.input_size,
# "vocab": dev_data.vocab,
# "tag_map": dev_data.tag_map,
# }
# testdata = {
# "batch_size": test_data.batch_size,
# "input_size": test_data.input_size,
# "vocab": test_data.vocab,
# "tag_map": test_data.tag_map,
# }
# if not os.path.exists("./model/train_data_map.pkl"):
# f = open("./model/train_data_map.pkl", "wb")
# pickle.dump(traindata, f)
# f.close()
# if not os.path.exists("./model/dev_data_map.pkl"):
# f = open("./model/dev_data_map.pkl", "wb")
# pickle.dump(devdata, f)
# f.close()
# if not os.path.exists("./model/test_data_map.pkl"):
# f = open("./model/test_data_map.pkl", "wb")
# pickle.dump(testdata, f)
# f.close()
# Use selected tagging scheme (IOB / IOBES)
#update_tag_scheme(train_sentences, FLAGS.tag_schema)
#update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# Create a dictionary and a mapping for tags
'''
_t:{'O': 869087, 'B-LOC': 16571, 'I-LOC': 22531, 'B-PER': 8144, 'I-PER': 15881, 'B-ORG': 9277, 'I-ORG': 37689, '[SEP]': 8, '[CLS]': 10}
id_to_tag:{0: 'O', 1: 'I-ORG', 2: 'I-LOC', 3: 'B-LOC', 4: 'I-PER', 5: 'B-ORG', 6: 'B-PER', 7: '[CLS]', 8: '[SEP]'}
tag_to_id:{'O': 0, 'I-ORG': 1, 'I-LOC': 2, 'B-LOC': 3, 'I-PER': 4, 'B-ORG': 5, 'B-PER': 6, '[CLS]': 7, '[SEP]': 8}
'''
tag_to_id = train_data.tag_map
id_to_tag = {v: k for k, v in tag_to_id.items()}
with open(FLAGS.map_file, "wb") as f:
pickle.dump([tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
'''
[['在', '这', '里', '恕', '弟', '不', '恭', '之', '罪', '，', '敢', '在', '尊', '前', '一', '诤', '：', '前', '人', '论',
'书', '，', '每', '曰', '“', '字', '字', '有', '来', '历', '，', '笔', '笔', '有', '出', '处', '”', '，', '细', '读', '公',
'字', '，', '何', '尝', '跳', '出', '前', '人', '藩', '篱', '，', '自', '隶', '变', '而', '后', '，', '直', '至', '明', '季',
'，', '兄', '有', '何', '新', '出', '？'], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1762, 6821, 7027, 2609, 2475, 679, 2621, 722,
5389, 8024, 3140, 1762, 2203, 1184, 671, 6420, 8038, 1184, 782, 6389, 741, 8024, 3680, 3288, 100, 2099, 2099, 3300,
3341, 1325, 8024, 5011, 5011, 3300, 1139, 1905, 100, 8024, 5301, 6438, 1062, 2099, 8024, 862, 2214, 6663, 1139,
1184, 782, 5974, 5075, 8024, 5632, 7405, 1359, 5445, 1400, 8024, 4684, 5635, 3209, 2108, 8024, 1040, 3300, 862,
3173, 1139, 8043, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
'''
# train_data = prepare_dataset(
# train_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
# )
# dev_data = prepare_dataset(
# dev_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
# )
# test_data = prepare_dataset(
# test_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower
# )
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data.data), len(dev_data.data), len(test_data.data)))
train_manager = BatchManager(ftraindata, FLAGS.batch_size)
dev_manager = BatchManager(fdevdata, FLAGS.batch_size)
test_manager = BatchManager(ftestdata, FLAGS.batch_size)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess,
model,
FLAGS.ckpt_path,
logger,
global_steps=step)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main() -> None:
"""
Program entry point. Parses command line arguments to decide which dataset and model to use.
:return: None.
"""
parse_command_line_arguments()
print_num_gpus_available()
gpu = tf.config.experimental.list_physical_devices(device_type='GPU')
tf.config.experimental.set_memory_growth(gpu[0], True)
# Start recording time.
start_time = time.time()
# Create label encoder.
l_e = create_label_encoder()
# Multiclass classification (mini-MIAS dataset)
if config.dataset == "mini-MIAS":
# Import entire dataset.
images, chars, labels = import_minimias_dataset(
data_dir="../data/{}/images".format(config.dataset),
label_encoder=l_e)
# Split dataset into training/test/validation sets (60%/20%/20% split).
X_train, X_test, y_train, y_test = dataset_stratified_split(
split=0.20, dataset=images, labels=labels)
X_train, X_val, y_train, y_val = dataset_stratified_split(
split=0.25, dataset=X_train, labels=y_train)
if config.SAMPLING == 'x':
X_train_rebalanced = X_train
y_train_rebalanced = y_train
else:
print(len(y_train))
print(l_e.classes_)
print(y_train.sum(axis=0))
if len(config.CLASS_TYPE.split('-')) == 2:
if config.SAMPLING == 'up':
X_train_rebalanced, y_train_rebalanced = generate_image_transforms_upsample(
X_train, y_train)
elif config.SAMPLING == 'down':
X_train_rebalanced, y_train_rebalanced = generate_image_transforms_downsample(
X_train, y_train)
if len(config.CLASS_TYPE.split(
'-')) != 2 and config.SAMPLING == 'up':
X_train_rebalanced, y_train_rebalanced = generate_image_transforms(
X_train, y_train)
print(len(y_train_rebalanced))
print(l_e.classes_)
print(y_train_rebalanced.sum(axis=0))
# Create and train CNN model.
if config.cnn == "ResNet":
if config.model == 'basic':
model = generate_resnet_model(l_e.classes_.size)
elif config.cnn == "VGG":
if config.model == 'basic':
model = generate_vgg_model(l_e.classes_.size)
else:
model = generate_vgg_model_advance(l_e.classes_.size)
if config.run_mode == "train":
model = train_network(l_e.classes_.size, model, X_train_rebalanced,
y_train_rebalanced, X_val, y_val,
config.BATCH_SIZE, config.EPOCH_1,
config.EPOCH_2)
# Binary classification (CBIS-DDSM dataset).
elif config.dataset == "CBIS-DDSM":
images, labels, density, cc, mlo = import_cbisddsm_training_dataset(
l_e)
images_test, labels_test, density_test, cc_test, mlo_test = import_cbisddsm_testing_dataset(
l_e)
if len(config.model.split('-')) > 1 and config.model.split(
'-')[1] == '3':
if config.run_mode == "train":
X = np.vstack((images, density, cc, mlo))
elif config.run_mode == "test":
X_test = np.vstack(
(images_test, density_test, cc_test, mlo_test))
X_test = X_test.transpose()
else:
if config.run_mode == "train":
X = np.vstack((images, density))
elif config.run_mode == "test":
X_test = np.vstack((images_test, density_test))
X_test = X_test.transpose()
if config.run_mode == "test":
y_test = labels_test
if config.run_mode == "train":
# Split training dataset into training/validation sets (75%/25% split).
X_train, X_val, y_train, y_val = dataset_stratified_split(
split=0.25, dataset=X.transpose(), labels=labels)
# X_train, X_val, y_train, y_val = dataset_stratified_split(split=0.25, dataset=images, labels=labels)
if config.run_mode == "train":
dataset_train = create_dataset(X_train, y_train)
dataset_val = create_dataset(X_val, y_val)
elif config.run_mode == "test":
dataset_test = create_dataset(X_test, y_test)
# Create and train CNN model.
if config.cnn == "ResNet":
if config.model.startswith('basic'):
if len(config.model.split('-')) == 1:
model = generate_resnet_model(l_e.classes_.size)
else:
model = generate_resnet_model_and_density(
l_e.classes_.size)
else:
if len(config.model.split('-')) == 1:
model = generate_resnet_model_advance(l_e.classes_.size)
else:
model = generate_resnet_model_advance_and_density(
l_e.classes_.size)
elif config.cnn == "VGG":
if config.model.startswith('basic'):
if len(config.model.split('-')) == 1:
model = generate_vgg_model(l_e.classes_.size)
else:
model = generate_vgg_model_and_density(l_e.classes_.size)
else:
if len(config.model.split('-')) == 1:
model = generate_vgg_model_advance(l_e.classes_.size)
else:
model = generate_vgg_model_advance_and_density(
l_e.classes_.size)
if config.run_mode == "train":
model = train_network(l_e.classes_.size, model, dataset_train,
None, dataset_val, None, config.BATCH_SIZE,
config.EPOCH_1, config.EPOCH_2)
else:
print_error_message()
try:
# Save the model
# model.save("../saved_models/dataset-{}_model-{}-{}_" + datetime.now().strftime("%d%Y%H%M%S") + ".h5".format(config.dataset, config.model, config.cnn))
if config.run_mode == "train":
save_time = datetime.now().strftime("%Y%m%d%H%M")
model.save_weights(
"/cs/tmp/sjc29/saved_models/dataset-{}_model-{}-{}_{}.h5".
format(config.dataset, config.model, config.cnn, save_time))
print('save_time: ', save_time)
print_runtime("Finish Training", round(time.time() - start_time,
2))
elif config.run_mode == "test":
model.load_weights(
"/cs/tmp/sjc29/saved_models/dataset-{}_model-{}-{}_{}.h5".
format(config.dataset, config.model, config.cnn,
config.MODEL_SAVE_TIME))
except:
print('save/load model error: ' + sys.exc_info()[0])
# print config
print_config()
# Evaluate model results.
if config.dataset == "mini-MIAS":
if config.run_mode == "train":
y_pred = make_predictions(model, X_val)
evaluate(y_val, y_pred, l_e, config.dataset, config.CLASS_TYPE,
'output')
print_runtime("Finish Prediction Validation Set",
round(time.time() - start_time, 2))
elif config.run_mode == "test":
y_pred_test = make_predictions(model, X_test)
evaluate(y_test, y_pred_test, l_e, config.dataset,
config.CLASS_TYPE, 'output_test')
print_runtime("Finish Prediction Testing Set",
round(time.time() - start_time, 2))
elif config.dataset == "CBIS-DDSM":
if config.run_mode == "train":
y_pred = make_predictions(model, dataset_val)
evaluate(y_val, y_pred, l_e, config.dataset, config.CLASS_TYPE,
'output')
print_runtime("Finish Prediction Validation Set",
round(time.time() - start_time, 2))
elif config.run_mode == "test":
y_pred_test = make_predictions(model, dataset_test)
evaluate(y_test, y_pred_test, l_e, config.dataset,
config.CLASS_TYPE, 'output_test')
print_runtime("Finish Prediction Testing Set",
round(time.time() - start_time, 2))
# Print the prediction
# print(y_pred)
# Print training runtime.
print_runtime("Total", round(time.time() - start_time, 2))
# Delay time in seconds for API call returns in case of auth
# failures (some DoS-countermeasure). For testing purposes we
# usually want this as fast as possible. This value gets used in
# create_mq_objects.py for the creation of the queue manager conn
# auth.
FAIL_DELAY = os.environ.get(
'PYMQI_TEST_QM_CONN_AUTH_FAIL_DELAY', '0')
# Queue naming setup
class QUEUE:
PREFIX = os.environ.get('PYMQI_TEST_QUEUE_PREFIX', '')
QUEUE_NAMES = {
'TestRFH2PutGet': PREFIX + 'TEST.PYMQI.RFH2PUTGET',
'TestQueueManager': PREFIX + 'TEST.PYMQI.QUEUEMANAGER',
}
# convenience attribute derived from above settings, may be used for tests
# that mandate the MQSERVER environment variable
# E.g. MQSERVER="SVRCONN.1/TCP/mq.example.org(1777)"
MQSERVER = '%(channel)s/%(transport)s/%(host)s(%(port)s)' % {
'channel': QM.CHANNEL,
'transport': QM.TRANSPORT,
'host': QM.HOST,
'port': QM.PORT,
}
if __name__ == '__main__':
utils.print_config(PATHS, MQ)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist, load data if exists maps
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# 设置训练日志目录
train_log = os.path.join(FLAGS.logdir, "train")
if not os.path.exists(train_log):
os.makedirs(train_log)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data # the nums of batch data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
# 观察所建立的计算图
train_writer = tf.summary.FileWriter(train_log, sess.graph)
logger.info("start training")
loss = []
dev_f1 = []
test_f1 = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss, merged = model.run_step(
sess, True, batch) # step是global step
# 在迭代中输出到结果
train_writer.add_summary(merged, step)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
# use dev data to validation the model
best, dev_f1_value = evaluate(sess, model, "dev", dev_manager,
id_to_tag, logger)
# store the dev f1
dev_f1.append(dev_f1_value)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
# use current the model to test
_, test_f1_value = evaluate(sess, model, "test", test_manager,
id_to_tag, logger)
# store the test f1
test_f1.append(test_f1_value)
# write the dev_f1 and test_f1 to file
f1_result = {}
f1_result["dev_f1"] = dev_f1
f1_result["test_f1"] = test_f1
write_data_to_file(f1_result, "f1_result")
def train():
# load data sets
train_sentences = load_sentences(
FLAGS.train_file, FLAGS.lower,
FLAGS.zeros) # dimension:num_sentence*len_sentence*2
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(
train_sentences,
FLAGS.tag_schema) # dimension:num_sentence*len_sentence*2
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb: # 如果使用预训练的词嵌入
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[
0] # dico_chars_train dimension: 训练数据集中出现的字符类别数*2，
dico_chars, char_to_id, id_to_char = augment_with_pretrained( # 利用测试数据样本集中的字对dico_chars_train进行补充
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f: # 创建map_file文件
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id,
FLAGS.lower) # dimension: NumSentence*4*LenSentence
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
train_manager = BatchManager(
train_data, FLAGS.batch_size
) # batch_data dimension: BatchNum*4*BatchSize*MaxLenSentence
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file): # 若已有config_file则读取加载
config = load_config(FLAGS.config_file)
else: # 若没有config_file则新建并保存为文件
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger) # 将config打印到日志文件
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True # 动态申请内存
steps_per_epoch = train_manager.len_data # len_data: ceil(NumSentence/BatchSize)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch): # 括号中数字是epoach数量
for batch in train_manager.iter_batch(
shuffle=True
): # 一次从batch_data中取出一个batch,Shuffle为True表示打乱batch_data的顺序
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
# View the tensorboard graph by running the following code and then going to the terminal and typing:
# tensorboard --logdir = tensorboard_logs
merged = tf.summary.merge_all()
if not os.path.exists('tensorboard_logs/'):
os.makedirs('tensorboard_logs/')
my_writer = tf.summary.FileWriter('tensorboard_logs/', sess.graph)
def train():
train_sentences, dico, char_to_id, id_to_char = load_sentence(
FLAGS.train_file)
if not os.path.isfile(FLAGS.map_file):
if FLAGS.pre_emb:
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico.copy(),
FLAGS.emb_file,
)
else:
sentences, dico, char_to_id, id_to_char = load_sentence(
FLAGS.train_file)
print(train_sentences[0])
with open(FLAGS.map_file, 'wb') as f:
pickle.dump([char_to_id, id_to_char], f)
else:
with open(FLAGS.map_file, 'rb') as f:
char_to_id, id_to_char = pickle.load(f)
train_data, test_data, dev_data = prepare_dataset(train_sentences,
char_to_id)
print(train_data[0])
print(test_data[0])
print(dev_data[0])
print(len(train_data), len(dev_data), len(test_data))
train_manager = BatchManager(train_data, FLAGS.batch_size)
test_manager = BatchManager(test_data, 100)
dev_manager = BatchManager(dev_data, 100)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
tf_config = tf.ConfigProto(gpu_options=gpu_options)
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
best = 0
# sess.graph.finalize()
for i in range(50):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{},".format(
iteration, step % steps_per_epoch, steps_per_epoch))
loss = []
Acc_result = evaluate(sess, model, "dev", dev_manager, logger)
logger.info("Acc{}".format(Acc_result))
logger.info("test")
# precision, recall, f1_score = model.evaluete_(sess,test_manager)
# logger.info("P, R, F,{},{},{}".format(precision, recall, f1_score))
test_result = evaluate(sess, model, "test", test_manager, logger)
if test_result > best:
best = test_result
save_model(sess, model, FLAGS.ckpt_path, logger)
dataset_masks_name = config.get('Paths', 'dataset_masks_name')
annotations_file_name = config.get('Paths', 'annotations_file_name')
final_annotations_file_name = config.get('Paths',
'final_annotations_file_name')
type_of_border = config.get('Annotations', 'type_of_border')
patch_size_height = config.getint('MultiScale', 'patch_size_height')
patch_size_width = config.getint('MultiScale', 'patch_size_width')
flip = config.getboolean('DataAugmentation', 'flip')
rotate = config.getboolean('DataAugmentation', 'rotate')
contrast = config.getboolean('DataAugmentation', 'contrast')
if display_logs:
print_config(config_path)
# Build the new dataset
create_dataset_hierarchy(path_dataset_created, augmented_dataset_name,
dataset_images_name, dataset_masks_name)
# Transfer original images
transfer_original_images(path_dataset_created, augmented_dataset_name,
dataset_name, dataset_images_name,
dataset_masks_name)
# Load the transformations
if flip:
data_augmentation_flip(path_dataset_created, augmented_dataset_name,
dataset_images_name, dataset_masks_name)
def train(X_train,X_dev,X_test):
# load data sets
train_sentences = X_train
dev_sentences = X_dev
test_sentences = X_test
train_sentences_loc = load_sentences(FLAGS.train_file_loc, FLAGS.lower, FLAGS.zeros)
dev_sentences_loc = load_sentences(FLAGS.dev_file_loc, FLAGS.lower, FLAGS.zeros)
test_sentences_loc = load_sentences(FLAGS.test_file_loc, FLAGS.lower, FLAGS.zeros)
train_sentences_org = load_sentences(FLAGS.train_file_org, FLAGS.lower, FLAGS.zeros)
dev_sentences_org = load_sentences(FLAGS.dev_file_org, FLAGS.lower, FLAGS.zeros)
test_sentences_org = load_sentences(FLAGS.test_file_org, FLAGS.lower, FLAGS.zeros)
train_sentences_per = load_sentences(FLAGS.train_file_per, FLAGS.lower, FLAGS.zeros)
dev_sentences_per = load_sentences(FLAGS.dev_file_per, FLAGS.lower, FLAGS.zeros)
test_sentences_per = load_sentences(FLAGS.test_file_per, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
update_tag_scheme(train_sentences_loc, FLAGS.tag_schema)
update_tag_scheme(test_sentences_loc, FLAGS.tag_schema)
update_tag_scheme(train_sentences_per, FLAGS.tag_schema)
update_tag_scheme(test_sentences_per, FLAGS.tag_schema)
update_tag_scheme(train_sentences_org, FLAGS.tag_schema)
update_tag_scheme(test_sentences_org, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences])
)
)
dico_chars_train_loc = char_mapping(train_sentences_loc, FLAGS.lower)[0]
dico_chars_loc, char_to_id_loc, id_to_char_loc = augment_with_pretrained(
dico_chars_train_loc.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences_loc])
)
)
dico_chars_train_per = char_mapping(train_sentences_per, FLAGS.lower)[0]
dico_chars_per, char_to_id_per, id_to_char_per = augment_with_pretrained(
dico_chars_train_per.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences_per])
)
)
dico_chars_train_org = char_mapping(train_sentences_org, FLAGS.lower)[0]
dico_chars_org, char_to_id_org, id_to_char_org = augment_with_pretrained(
dico_chars_train_org.copy(),
FLAGS.emb_file,
list(itertools.chain.from_iterable(
[[w[0] for w in s] for s in test_sentences_org])
)
)
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
_c_loc, char_to_id_loc, id_to_char_loc = char_mapping(train_sentences_loc, FLAGS.lower)
_c_per, char_to_id_per, id_to_char_per = char_mapping(train_sentences_per, FLAGS.lower)
_c_org, char_to_id_org, id_to_char_org = char_mapping(train_sentences_org, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
_t_loc, tag_to_id_loc, id_to_tag_loc = tag_mapping(train_sentences_loc)
_t_per, tag_to_id_per, id_to_tag_per = tag_mapping(train_sentences_per)
_t_org, tag_to_id_org, id_to_tag_org = tag_mapping(train_sentences_org)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag,char_to_id_loc, id_to_char_loc, tag_to_id_loc, id_to_tag_loc,char_to_id_per, id_to_char_per, tag_to_id_per, id_to_tag_per,char_to_id_org, id_to_char_org, tag_to_id_org, id_to_tag_org], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag,char_to_id_loc, id_to_char_loc, tag_to_id_loc, id_to_tag_loc,char_to_id_per, id_to_char_per, tag_to_id_per, id_to_tag_per,char_to_id_org, id_to_char_org, tag_to_id_org, id_to_tag_org = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(
train_sentences, char_to_id, tag_to_id, FLAGS.lower
)
dev_data = prepare_dataset(
dev_sentences, char_to_id, tag_to_id, FLAGS.lower
)
test_data = prepare_dataset(
test_sentences, char_to_id, tag_to_id, FLAGS.lower
)
print("%i / %i / %i sentences in train / dev / test." % (
len(train_data),len(dev_data), len(test_data)))
train_data_loc = prepare_dataset_ner(
train_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower
)
dev_data_loc = prepare_dataset_ner(
dev_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower
)
test_data_loc = prepare_dataset_ner(
test_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower
)
print("%i / %i / %i sentences_loc in train / dev / test." % (
len(train_data_loc), len(dev_data_loc), len(test_data_loc)))
train_data_per = prepare_dataset_ner(
train_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower
)
dev_data_per = prepare_dataset_ner(
dev_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower
)
test_data_per = prepare_dataset_ner(
test_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower
)
print("%i / %i / %i sentences_per in train / dev / test." % (
len(train_data_per), len(dev_data_per), len(test_data_per)))
train_data_org = prepare_dataset_ner(
train_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower
)
dev_data_org = prepare_dataset_ner(
dev_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower
)
test_data_org = prepare_dataset_ner(
test_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower
)
print("%i / %i / %i sentences_org in train / dev / test." % (
len(train_data_org), len(dev_data_org), len(test_data_org)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
train_manager_loc = BatchManager(train_data_loc, FLAGS.batch_size)
train_manager_per = BatchManager(train_data_per, FLAGS.batch_size)
train_manager_org = BatchManager(train_data_org, FLAGS.batch_size)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id,char_to_id_loc, tag_to_id_loc,char_to_id_per, tag_to_id_per,char_to_id_org, tag_to_id_org)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
steps_per_epoch_loc = train_manager_loc.len_data
steps_per_epoch_per = train_manager_per.len_data
steps_per_epoch_org = train_manager_org.len_data
model = create_model(Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, id_to_char_loc, id_to_char_per, id_to_char_org, logger)
with tf.Session(config=tf_config, graph = model.graph ) as sess:
sess.run(tf.global_variables_initializer())
if config["pre_emb"]:
emb_weights = sess.run(model.char_lookup.read_value())
emb_weights_ner = sess.run(model.char_lookup.read_value())
emb_weights, emb_weights_ner = load_word2vec(config["emb_file"], id_to_char, id_to_char_loc,id_to_char_per,id_to_char_org, config["char_dim"],
emb_weights, emb_weights_ner)
sess.run(model.char_lookup.assign(emb_weights))
logger.info("Load pre-trained embedding.")
logger.info("start training")
loss = []
loss_loc = []
loss_per = []
loss_org = []
for i in range(100):
for batch_loc in train_manager_loc.iter_batch(shuffle=True):
step_loc, batch_loss_loc = model.run_step_ner(sess, True, batch_loc)
loss_loc.append(batch_loss_loc)
if step_loc % FLAGS.steps_check == 0:
iteration_loc = step_loc // steps_per_epoch_loc + 1
logger.info("iteration:{} step_loc:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration_loc, step_loc % steps_per_epoch_loc, steps_per_epoch_loc, np.mean(loss_loc)))
loss_loc = []
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration_1 = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"SKILL loss:{:>9.6f}".format(
iteration_1, step % steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
precision_loc_dev = model.precision(sess, dev_manager, id_to_tag)
precision_loc_test = model.precision(sess, test_manager, id_to_tag)
for batch_per in train_manager_per.iter_batch(shuffle=True):
step_per, batch_loss_per = model.run_step_ner(sess, True, batch_per)
loss_per.append(batch_loss_per)
if step_per % FLAGS.steps_check == 0:
iteration_per = step_per // steps_per_epoch_per + 1
logger.info("iteration:{} step_per:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration_per, step_per % steps_per_epoch_per, steps_per_epoch_per, np.mean(loss_per)))
loss_per = []
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration_2 = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"SKILL loss:{:>9.6f}".format(
iteration_2, step % steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
precision_per_dev = model.precision(sess, dev_manager, id_to_tag)
precision_per_test = model.precision(sess, test_manager, id_to_tag)
for batch_org in train_manager_org.iter_batch(shuffle=True):
step_org, batch_loss_org = model.run_step_ner(sess, True, batch_org)
loss_org.append(batch_loss_org)
if step_org % FLAGS.steps_check == 0:
iteration_org = step_org // steps_per_epoch_org + 1
logger.info("iteration:{} step_org:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration_org, step_org % steps_per_epoch_org, steps_per_epoch_org, np.mean(loss_org)))
loss_org = []
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration_3 = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"SKILL loss:{:>9.6f}".format(
iteration_3, step % steps_per_epoch, steps_per_epoch, np.mean(loss)))
loss = []
precision_org_dev = model.precision(sess, dev_manager, id_to_tag)
precision_org_test = model.precision(sess, test_manager, id_to_tag)
best = evaluate(sess, model, "dev", dev_manager, id_to_tag,precision_loc_dev,precision_per_dev,precision_org_dev, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
best_test,results= evaluate(sess, model, "test", test_manager, id_to_tag,precision_loc_test,precision_per_test,precision_org_test, logger)
with open("CDTL_PSE-result.csv", "a",encoding='utf-8')as st_re:
st_re.write(str(results).replace("[", "").replace("]", ""))
st_re.write("\n")
help='disable GPU training')
args = parser.parse_args()
args.bi = not args.no_bi
args.cuda = torch.cuda.is_available() and not args.no_cuda
if args.cuda:
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
sorted_gpu_info = get_gpu_memory_map()
for gpu_id, (mem_left, util) in sorted_gpu_info:
if mem_left >= 1000:
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
# args.cuda = f'cuda:{gpu_id}'
print('use gpu:{} with {} MB left, util {}%'.format(gpu_id, mem_left, util))
break
else:
print(f'no gpu has memory left >= {args.min_mem} MB, exiting...')
exit()
else:
torch.set_num_threads(10)
args.data = 'train010203x_test04'
args.remark = 'del'
remark = f'_{args.data}_{args.remark}'
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
args.path = os.path.join('saved_model/debug_ext_ml', current_time + remark)
print_config(args)
os.environ["DATA"] = f"./data/DUC-json/{args.data}"
DATA_DIR = os.environ['DATA']
main(args)
def train():
# load data sets
# train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros)
# dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
all_train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
train_sentences, dev_sentences = split_train_dev(all_train_sentences)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# update_tag_scheme(dev_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
# dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars_train = char_mapping(all_train_sentences,
FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(all_train_sentences,
FLAGS.lower)
# _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(all_train_sentences)
# _t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# nlp = StanfordCoreNLP(r'E:\DC\dataset\泰一指尚评测数据\stanford-corenlp-full-2017-06-09')
#l_sorted_lexcion = load_lexcion(FLAGS.lexcion_file, nlp)
l_sorted_lexcion = []
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
l_sorted_lexcion, FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), len(dev_data), len(test_data)))
max_len = max(
[len(sentence[0]) for sentence in train_data + test_data + dev_data])
train_manager = BatchManager(train_data, FLAGS.batch_size, max_len)
dev_manager = BatchManager(dev_data, 800, max_len)
test_manager = BatchManager(test_data, 800, max_len)
# random.shuffle(train_data)
# pad_test_data = pad_data(test_data)
# pad_dev_data = pad_data(dev_data)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id, max_len)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
random.shuffle(train_data)
pad_train_data = pad_data(train_data, max_len)
strings, chars, lexcion_teatures, pos_ids, dep_ids, head_ids, targets = pad_train_data
for j in range(0, len(strings), FLAGS.batch_size):
batch = [
strings[j:j + FLAGS.batch_size],
chars[j:j + FLAGS.batch_size],
lexcion_teatures[j:j + FLAGS.batch_size],
pos_ids[j:j + FLAGS.batch_size],
dep_ids[j:j + FLAGS.batch_size],
head_ids[j:j + FLAGS.batch_size],
targets[j:j + FLAGS.batch_size]
]
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"AS loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger, i)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train():
# load data sets
train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower,
FLAGS.zeros)
dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros)
test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros)
# Use selected tagging scheme (IOB / IOBES)
update_tag_scheme(train_sentences, FLAGS.tag_schema)
update_tag_scheme(test_sentences, FLAGS.tag_schema)
# create maps if not exist
if not os.path.isfile(FLAGS.map_file):
# create dictionary for word
if FLAGS.pre_emb:
dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0]
dico_chars, char_to_id, id_to_char = augment_with_pretrained(
dico_chars_train.copy(), FLAGS.emb_file,
list(
itertools.chain.from_iterable([[w[0] for w in s]
for s in test_sentences])))
else:
_c, char_to_id, id_to_char = char_mapping(train_sentences,
FLAGS.lower)
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sentences)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# prepare data, get a collection of list containing index
train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id,
FLAGS.lower)
test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id,
FLAGS.lower)
print("%i / %i / %i sentences in train / dev / test." %
(len(train_data), 0, len(test_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
test_manager = BatchManager(test_data, 100)
# make path for store log and model if not exist
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = os.path.join("log", FLAGS.log_file)
logger = get_logger(log_path)
print_config(config, logger)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(100):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
#best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger)
#if best:
save_model(sess, model, FLAGS.ckpt_path, logger)
def train_new():
train_sent = load_sentences(FLAGS.filepath)
update_tag_scheme(train_sent, FLAGS.tag_schema)
if not os.path.isfile(FLAGS.map_file):
_c, char_to_id, id_to_char = char_mapping(train_sent, FLAGS.lower)
print("random embedding")
# Create a dictionary and a mapping for tags
_t, tag_to_id, id_to_tag = tag_mapping(train_sent)
with open(FLAGS.map_file, "wb") as f:
pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f)
else:
with open(FLAGS.map_file, "rb") as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
# 数据准备，划分验证集和训练集
np.random.seed(10)
train_sent_ = np.array(train_sent)
shuffle_indices = np.random.permutation(np.arange(len(train_sent)))
sent_shuffled = train_sent_[shuffle_indices]
dev_sample_index = -1 * int(FLAGS.dev_percentage * float(len(train_sent)))
train_sent_new, dev_sent = sent_shuffled[:dev_sample_index], sent_shuffled[
dev_sample_index:]
train_data = prepare_dataset(train_sent_new, char_to_id, tag_to_id,
FLAGS.lower)
dev_data = prepare_dataset(dev_sent, char_to_id, tag_to_id, FLAGS.lower)
print("%i / %i sentences in train." % (len(train_data), len(dev_data)))
train_manager = BatchManager(train_data, FLAGS.batch_size)
dev_manager = BatchManager(dev_data, 100)
make_path(FLAGS)
if os.path.isfile(FLAGS.config_file):
config = load_config(FLAGS.config_file)
else:
config = config_model(char_to_id, tag_to_id)
save_config(config, FLAGS.config_file)
make_path(FLAGS)
log_path = FLAGS.log_file
logger = get_logger(log_path)
print_config(config, logger)
# 根据需求，设置动态使用GPU资源
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
steps_per_epoch = train_manager.len_data
with tf.Session(config=tf_config) as sess:
fig = plt.figure()
ax = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
plt.grid(True)
plt.ion()
model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec,
config, id_to_char, logger)
logger.info("start training")
loss = []
for i in range(FLAGS.max_epoch):
for batch in train_manager.iter_batch(shuffle=True):
step, batch_loss = model.run_step(sess, True, batch)
loss.append(batch_loss)
if step % 20 == 0:
ax.scatter(step, np.mean(loss), c='b', marker='.')
plt.pause(0.001)
if step % FLAGS.steps_check == 0:
iteration = step // steps_per_epoch + 1
logger.info("iteration:{} step:{}/{}, "
"NER loss:{:>9.6f}".format(
iteration, step % steps_per_epoch,
steps_per_epoch, np.mean(loss)))
loss = []
best, f1 = evaluate(sess, model, "dev", dev_manager, id_to_tag,
logger)
ax2.scatter(i + 1, f1, c='b', marker='.')
plt.pause(0.001)
if best:
save_model(sess, model, FLAGS.ckpt_path, logger, "best")
