def evaluate_predictsamples(): config = load_config(FLAGS.config_file) logger = get_logger('log/test.log') print_config(config, logger) # limit GPU memory logger.info("start predict") tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) index = 1 with open('data/predict.txt', 'w', encoding='utf-8') as f_write: read_file = 'data/test.txt' for line in open(read_file, 'r', encoding='utf-8-sig').readlines(): result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result) content = str(index) + ',' if result is not None: entities = result['entities'] for entity in entities: if entity: print(entity['word'] + '\t' + entity['type']) # content += entity['word'] + '\t' + str(entity['start']) + '\t' + str(entity['end']) + '\t' + entity['type']+ ';' content += entity['word'] + '\t' + entity[ 'type'] + ';' f_write.write(content + '\n') index += 1
def main(): # read arguments args = parse_arguments() args['device'] = torch.device('cuda') #args['device'] = torch.device('cpu') load_config(args) print_config(args) # load data data = load_data(args) train_data, test_data = data args['spect_dim'] = train_data.size()[-1] args['time_step'] = train_data.size()[-2] # create model #model_low, model_high, joint_model_low, joint_model_high = create_model(args) #model_low = model_low.to(args['device']) #model_high = model_high.to(args['device']) #joint_model_low = joint_model_low.to(args['device']) #joint_model_high = joint_model_high.to(args['device']) model = create_model(args) model = model.to(args['device']) if args['action'] == 'train': # initialize training latents and train model train_latents, train_loss, best_loss, best_epoch, best_model = optimize_network(args, model, train_data, 'train') model = best_model.to(args['device']) test_loss, final_loss, final_pred, test_latents = optimize_network(args, model, test_data, 'test') elif args['action'] == 'test': # initialize testing latents and test model test_loss, final_loss, final_pred, test_latents = optimize_network(args, model, test_data, 'test') # save statistics basedir = "/projects/dataset_processed/xiaoyam" #basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) model_folder = os.path.join(basedir, 'model_saves', 'pytorch', args['config_folder'], 'config_' + str(args['config_num'])) if not os.path.isdir(model_folder): os.makedirs(model_folder) save_results(args, model_folder, { 'train_latents': train_latents, 'test_latents': test_latents, 'test_loss': test_loss, 'final_test_loss': final_loss, 'train_loss': train_loss, 'final_pred': final_pred, 'best_train_loss': best_loss, 'best_train_epoch': best_epoch, 'best_model': model, 'spect_true': test_data, })
def main(): # add argumentation parser = argparse.ArgumentParser( description='MobileNet_v2_DeepLab_v3 Pytorch Implementation') parser.add_argument( '--dataset', default='cityscapes', choices=['cityscapes', 'other'], help='Dataset used in training MobileNet v2+DeepLab v3') parser.add_argument('--root', default='./data/cityscapes', help='Path to your dataset') parser.add_argument('--epoch', default=None, help='Total number of training epoch') parser.add_argument('--lr', default=None, help='Base learning rate') parser.add_argument('--pretrain', default=None, help='Path to a pre-trained backbone model') parser.add_argument('--resume_from', default=None, help='Path to a checkpoint to resume model') args = parser.parse_args() params = Params() # parse args if not os.path.exists(args.root): if params.dataset_root is None: raise ValueError('ERROR: Root %s not exists!' % args.root) else: params.dataset_root = args.root if args.epoch is not None: params.num_epoch = args.epoch if args.lr is not None: params.base_lr = args.lr if args.pretrain is not None: params.pre_trained_from = args.pretrain if args.resume_from is not None: params.resume_from = args.resume_from LOG('Network parameters:') print_config(params) # create dataset and transformation LOG('Creating Dataset and Transformation......') datasets = create_dataset(params) LOG('Creation Succeed.\n') # create model LOG('Initializing MobileNet and DeepLab......') net = MobileNetv2_DeepLabv3(params, datasets) LOG('Model Built.\n') # let's start to train! net.Train() net.Test()
def __init__(self,config): self.config = config print_config(config) # build session sess_config = tf.ConfigProto(device_count = {"CPU":self.config.num_cpu}, inter_op_parallelism_threads=0, intra_op_parallelism_threads=0) self.sess = tf.Session(config=sess_config)
def do_config(self, arg): """View or change the configuration.""" # TODO if not arg: utils.print_config() return arg = self.split_args() print(arg) print("*** Not implemented yet.")
def main(): # read arguments args = parse_arguments() args['device'] = torch.device('cuda') #args['device'] = torch.device('cpu') load_config(args) print_config(args) # load data data = load_data(args) (train_spect, train_pcm), (test_spect, test_pcm) = data args['spect_dim'] = train_spect.size()[-1] args['pcm_dim'] = train_pcm.size()[-1] # create model model = create_model(args) model = model.to(args['device']) if args['action'] == 'train': # initialize training latents and train model train_latents, train_loss, best_loss, best_epoch, best_model = optimize_network( args, model, train_spect, train_pcm, 'train') model = best_model.to(args['device']) test_loss, final_loss, final_pred, test_latents = optimize_network( args, model, test_spect, test_pcm, 'test') elif args['action'] == 'test': # initialize testing latents and test model test_loss, final_loss, final_pred, test_latents = optimize_network( args, model, test_spect, test_pcm, 'test') # save statistics basedir = os.getcwd() #basedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) model_folder = os.path.join(basedir, 'model_saves', 'pytorch', args['config_folder'], 'config_' + str(args['config_num'])) if not os.path.isdir(model_folder): os.makedirs(model_folder) save_results( args, model_folder, { 'train_latents': train_latents, 'test_latents': test_latents, 'test_loss': test_loss, 'final_test_loss': final_loss, 'train_loss': train_loss, 'final_pred': final_pred, 'best_train_loss': best_loss, 'best_train_epoch': best_epoch, 'best_model': model, 'spect_true': test_spect })
def main(): # add argumentation parser = argparse.ArgumentParser(description='MobileNet_v2_DeepLab_v3 Pytorch Implementation') #todo maybe make it work with multiple datasets? #parser.add_argument('--dataset', default='cityscapes', choices=['cityscapes', 'other'], # help='Dataset used in training MobileNet v2+DeepLab v3') parser.add_argument('--root', default='./data/cityscapes', help='Path to your dataset') parser.add_argument('--epoch', default=None, help='Total number of training epoch') parser.add_argument('--lr', default=None, help='Base learning rate') parser.add_argument('--pretrain', default=None, help='Path to a pre-trained backbone model') parser.add_argument('--resume_from', default=None, help='Path to a checkpoint to resume model') parser.add_argument('--logdir', default=None, help='Directory to save logs for Tensorboard') parser.add_argument('--batch_size', default=128, help='Batch size for training') args = parser.parse_args() params = Params() # parse args if not os.path.exists(args.root): if params.dataset_root is None: raise ValueError('ERROR: Root %s doesn\'t exist!' % args.root) else: params.dataset_root = args.root if args.epoch is not None: params.num_epoch = int(args.epoch) if args.lr is not None: params.base_lr = args.lr if args.pretrain is not None: params.pre_trained_from = args.pretrain if args.resume_from is not None: params.resume_from = args.resume_from if args.logdir is not None: params.logdir = args.logdir params.summary_dir, params.ckpt_dir = create_train_dir(params.logdir) params.train_batch = int(args.batch_size) LOG('Network parameters:') print_config(params) # create dataset and transformation LOG('Creating Dataset and Transformation......') datasets = create_dataset(params) LOG('Creation Succeed.\n') # create model LOG('Initializing MobileNet and DeepLab......') net = MobileNetv2_DeepLabv3(params, datasets) LOG('Model Built.\n') # let's start to train! net.Train() net.Test()
def train_model(model_module, config, params=None, values=None, do_evaluate=False): if params is not None: for param, value in zip(params, values): assert param in config.__dict__ setattr(config, param, value) print_config(config) data_builder = model_module.DataBuilder(config) data_builder.run() config_holder = model_module.ConfigHolder(config) model = model_module.Model(config_holder) model.build() train = model_module.CoNLLDataset(config_holder.filename_train, config_holder.processing_word_train, config_holder.processing_tag, config_holder.max_iter, use_buckets=config.bucket_train_data, batch_size=config.batch_size, shuffle=config.shuffle_train_data, sort=config.sort_train_data) test = model_module.CoNLLDataset(config_holder.filename_dev, config_holder.processing_word_infer, config_holder.processing_tag, config_holder.max_iter, sort=True) train_eval = model_module.CoNLLDataset(config_holder.filename_train, config_holder.processing_word_infer, config_holder.processing_tag, sort=True, max_iter=config_holder.train_sentences_to_eval) model.train(train, test, train_eval) model.close_session() tf.reset_default_graph() # read accuracies for all iterations df = pd.read_csv(config_holder.training_log, names=['epoch', 'acc_train', 'acc_test', 'train_loss', 'nbatches', 'epoch_time', 'train_time', 'eval_time']) acc_train_list = df['acc_train'] acc_test_list = df['acc_test'] train_loss_list = df['train_loss'] # evaluate if do_evaluate is True: print("Evaluating...") evaluate(model, config_holder, config_holder.filename_dev, 'LANG_NA', 'dev', config_holder.out_dir) model.close_session() tf.reset_default_graph() return acc_train_list, acc_test_list, train_loss_list
def get_config(self): """ 从模型参数配置文件中获取参数或者用config_model函数生成参数并存储 :return:日志logger及参数列表config """ make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = self.config_model(self.char_to_id, self.tag_to_id) save_config(config, FLAGS.config_file) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) return logger, config
def main(args): # print config utils.print_config(args) if args.config: return id_file_path = args.id_file overwrite_ids = args.update_ids dont_have_id_file = not os.path.exists(id_file_path) data_file_path = args.output_file update_data = args.update_data dont_have_data = not os.path.exists(data_file_path) download_attachments = args.download add_to_ipfs = args.ipfs add_to_qri = args.qri #------------------------------------------------------------------- if overwrite_ids or dont_have_id_file: fetch_valid_id_list(args) else: print("...using existing id list '{}'".format(id_file_path)) #------------------------------------------------------------------- if update_data or dont_have_data: extract_data_from_pages(args) else: print("...using existing data file '{}'".format(data_file_path)) #------------------------------------------------------------------- if download_attachments: fetch_attachment_files(args) else: print("...skipping attachment download") #------------------------------------------------------------------- if add_to_ipfs: add_to_ipfs_and_save_results(args) else: print("...skipping IPFS add") #------------------------------------------------------------------- if add_to_qri: data_file_path = args.output_file.replace(".json", "_with_hashes.json") utils.add_or_save_to_qri(args.dsname, data_file_path, "templates/structure.json", "templates/meta.json") else: print("...skipping QRI add")
def first_menu(): menu = -1 while menu == -1 or menu == 1: try: menu = int( input(""" 1. Print current config. 2. Config interfaces. 3. Choose competition phase (Firewall rules). 0. Quit. """)) if menu == 1: print_config() elif menu == 2: change_config() create_netplan_config_interactive(if_list) elif menu == 3: second_menu() elif (menu) == 0: sys.exit else: print("Your choice (%d) is wrong. Please, try again." % (menu)) except ValueError: print('ERRORE di input.')
elif config['hyperparameters']['lr_multistep'] != 'None': tmp_str = "lr_scheduler : [milestones: " for milestone in scheduler_for_lr.milestones: tmp_str = tmp_str + str(milestone) + ', ' tmp_str += ' gamma: ' tmp_str += str(scheduler_for_lr.gamma) tmp_str += ']' print(tmp_str) print("Size of batch : " + str(train_loader.batch_size)) print("transform : " + str(transform)) print("num. train data : " + str(len(train_dataset))) print("num. valid data : " + str(len(valid_dataset))) print("num_classes : " + str(num_classes)) print("classes : " + str(encoder.dict)) utils.print_config(config) input("Press any key to continue..") def train(epoch): crnn.train() generator_g.train() discriminator_ga.train() discriminator_fa.train() train_loss = 0 global global_iter_train batch_size = train_loader.batch_size with torch.set_grad_enabled(True):
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences]) ) ) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset( train_sentences, char_to_id, tag_to_id, FLAGS.lower ) dev_data = prepare_dataset( dev_sentences, char_to_id, tag_to_id, FLAGS.lower ) test_data = prepare_dataset( test_sentences, char_to_id, tag_to_id, FLAGS.lower ) print("%i / %i / %i sentences in train / dev / test." % ( len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): #print batch step, batch_loss = model.run_step(sess, True, batch) #print step loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train(): # load data sets datasets = load_sentences(FLAGS.train_file, FLAGS.lower) random.shuffle(datasets) train_sentences = datasets[:14000] test_sentences = datasets[14000:] # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word char_to_id, _ = elmo_char_mapping(FLAGS.elmo_vocab) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i sentences in train / dev." % (len(train_data), len(test_data))) elmo_batcher = get_batcher() train_manager = BatchManager(train_data, FLAGS.batch_size, elmo_batcher) test_manager = BatchManager(test_data, FLAGS.batch_size, elmo_batcher) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto(allow_soft_placement=True) tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: elmo_model = load_elmo() model = create_model(sess, Model, FLAGS.ckpt_path, elmo_model, config, logger) logger.info("start training") loss = [] for i in range(FLAGS.max_epoch): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info( "iteration:{} step:{}/{}, NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "test", test_manager, id_to_tag, logger) # evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger)
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) #update_tag_scheme(train_sentences, FLAGS.tag_schema) #update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset( train_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower ) dev_data = prepare_dataset( dev_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower ) test_data = prepare_dataset( test_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower ) print("%i / %i / %i sentences in train / dev / test." % ( len(train_data), 0, len(test_data))) train_len=len(train_data) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, FLAGS.batch_size) test_manager = BatchManager(test_data, FLAGS.batch_size) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, config, logger) logger.info("start training") loss = [] for i in range(FLAGS.max_epoch): from tqdm import tqdm for batch in tqdm(train_manager.iter_batch(shuffle=True)): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step%steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] print("save result epoch:",i," ***************************************************") best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger,i) if i>=8: save_model(sess, model, FLAGS.ckpt_path, logger, global_steps=step) evaluate(sess, model, "test", test_manager, id_to_tag, logger,i)
def train(): # 加载数据集 train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # 选择tag schema(IOB / IOBES) I:中间,O:其他,B:开始 | E:结束,S:单个 update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) update_tag_scheme(dev_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # 配置文件:char_to_id, id_to_char, tag_to_id, id_to_tag的数据 # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences]) ) ) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences, FLAGS.id_to_tag_path, FLAGS.tag_to_id_path) # with open('maps.txt','w',encoding='utf8') as f1: # f1.writelines(str(char_to_id)+" "+id_to_char+" "+str(tag_to_id)+" "+id_to_tag+'\n') with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # # prepare data, get a collection of list containing index # train_data[0][0]:一句话; # train_data[0][1]:单个字的编号; # train_data[0][2]:切词之后,切词特征:词的大小是一个字的话是0,词的大小是2以上的话:1,2....,2,3; # train_data[0][3]:每个字的标签 train_data = prepare_dataset( train_sentences, char_to_id, tag_to_id, FLAGS.lower ) dev_data = prepare_dataset( dev_sentences, char_to_id, tag_to_id, FLAGS.lower ) test_data = prepare_dataset( test_sentences, char_to_id, tag_to_id, FLAGS.lower ) print("%i / %i / %i sentences in train / dev / test." % ( len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) # 按batch size将数据拆分 dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] # tf.device("/cpu:0") 指定运行的GPU(默认为GPU:0) with tf.device("/cpu:0"): for i in range(100): # 按批次训练模型。这个是训练的开始,可以从这里倒着找整个网络怎么训练 for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) # 打印信息: # iteration:迭代次数,也就是经过多少个epoch; # if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] # best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if i % 7 == 0: save_model(sess, model, FLAGS.ckpt_path, logger)
#长度不足补0 train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # GPU设置 tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.model_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0:
def main(): # load data sets global args args = parser.parse_args() pp.pprint(vars(args)) # running_name = 'X' use_cuda = cuda_model.ifUseCuda(args.gpu_id, args.multiGpu) # use_cuda = False # train_file = 'data/example.train' # dev_file = 'data/example.dev' test_file = 'data/example.test' # embedding_file = 'data/vec.txt' map_file = 'map.pkl' # config_file = 'config_file_pytorch' tag_file = 'tag.pkl' # embedding_easy_file = 'data/easy_embedding.npy' # train_sentences = load_sentences(train_file) # dev_sentences = load_sentences(dev_file) test_sentences = load_sentences(test_file) # train_sentences = dev_sentences # update_tag_scheme(train_sentences, args.tag_schema) update_tag_scheme(test_sentences, args.tag_schema) # update_tag_scheme(dev_sentences, args.tag_schema) if not os.path.isfile(tag_file): print("Tag file {:s} Not found".format(tag_file)) sys.exit(-1) else: with open(tag_file, 'rb') as t: tag_to_id, id_to_tag = pickle.load(t) if not os.path.isfile(map_file): print("Map file {:s} Not found".format(map_file)) # create dictionary for word # dico_chars_train = char_mapping(train_sentences)[0] # dico_chars, char_to_id, id_to_char = augment_with_pretrained( # dico_chars_train.copy(), # embedding_file, # list(itertools.chain.from_iterable( # [[w[0] for w in s] for s in test_sentences]) # ) # ) # # _, tag_to_id, id_to_tag = tag_mapping(train_sentences) # # with open(map_file, "wb") as f: # pickle.dump([char_to_id, id_to_char], f) else: with open(map_file, "rb") as f: char_to_id, id_to_char = pickle.load(f) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id) print("{:d} sentences in test.".format(len(test_data))) test_manager = BatchManager(test_data, 1) save_places = dir_utils.save_places(args.eval) # log_path = os.path.join("log", FLAGS.log_file) logger = get_logger( os.path.join(save_places.log_save_dir, 'evaluation-{:d}.txt'.format(args.fileid))) config = config_model(char_to_id, tag_to_id, args) print_config(config, logger) logger.info("start training") #Update: create model and embedding! model = NERModel.CNERPointer(char_dim=args.char_dim, seg_dim=args.seg_dim, hidden_dim=args.hidden_dim, max_length=15, output_classes=4, dropout=args.dropout, embedding_path=None, id_to_word=id_to_char, easy_load=None) print("Number of Params\t{:d}".format( sum([p.data.nelement() for p in model.parameters()]))) #Update: this won't work! # model = cuda_model.convertModel2Cuda(model, gpu_id=args.gpu_id, multiGpu=args.multiGpu) if use_cuda: model = model.cuda() model.eval() if args.eval is not None: # if os.path.isfile(args.resume): ckpt_filename = os.path.join( save_places.model_save_dir, 'checkpoint_{:04d}.pth.tar'.format(args.fileid)) assert os.path.isfile( ckpt_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_filename, map_location=lambda storage, loc: storage) model.load_state_dict(checkpoint['state_dict'], strict=True) train_iou = checkpoint['IoU'] print("=> loading checkpoint '{}', current iou: {:.04f}".format( ckpt_filename, train_iou)) ner_results = evaluate(model, test_manager, id_to_tag, use_cuda, max_len=5) eval_lines = test_ner(ner_results, save_places.summary_save_dir) for line in eval_lines: logger.info(line) f1 = float(eval_lines[1].strip().split()[-1]) return f1
# Delay time in seconds for API call returns in case of auth # failures (some DoS-countermeasure). For testing purposes we # usually want this as fast as possible. This value gets used in # create_mq_objects.py for the creation of the queue manager conn # auth. FAIL_DELAY = os.environ.get('PYMQI_TEST_QM_CONN_AUTH_FAIL_DELAY', '0') # Queue naming setup class QUEUE: PREFIX = os.environ.get('PYMQI_TEST_QUEUE_PREFIX', '') QUEUE_NAMES = { 'TestRFH2PutGet': PREFIX + 'TEST.PYMQI.RFH2PUTGET', 'TestQueueManager': PREFIX + 'TEST.PYMQI.QUEUEMANAGER', } # convenience attribute derived from above settings, may be used for tests # that mandate the MQSERVER environment variable # E.g. MQSERVER="SVRCONN.1/TCP/mq.example.org(1777)" MQSERVER = '%(channel)s/%(transport)s/%(host)s(%(port)s)' % { 'channel': QM.CHANNEL, 'transport': QM.TRANSPORT, 'host': QM.HOST, 'port': QM.PORT, } if __name__ == '__main__': utils.print_config(PATHS, MQ)
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) # "sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'I-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'I-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'I-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'I-TYPE']]" dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) # print("train_sentences[0]:{}".format(train_sentences[0])) # "train_sentences[0]:[['我', 'O'], ['要', 'O'], ['看', 'O'], ['乌', 'B-SLOC'], ['鲁', 'I-SLOC'], ['木', 'I-SLOC'], ['齐', 'I-SLOC'], ['市', 'I-SLOC'], ['第', 'I-SLOC'], ['四', 'I-SLOC'], ['十', 'I-SLOC'], ['九', 'I-SLOC'], ['中', 'I-SLOC'], ['学', 'I-SLOC'], ['东', 'I-SLOC'], ['门', 'E-SLOC'], ['去', 'O'], ['乌', 'B-ELOC'], ['鲁', 'I-ELOC'], ['木', 'I-ELOC'], ['齐', 'I-ELOC'], ['推', 'I-ELOC'], ['拿', 'I-ELOC'], ['职', 'I-ELOC'], ['业', 'I-ELOC'], ['学', 'I-ELOC'], ['校', 'I-ELOC'], ['南', 'I-ELOC'], ['门', 'E-ELOC'], ['沿', 'O'], ['西', 'B-ROAD'], ['虹', 'I-ROAD'], ['东', 'I-ROAD'], ['路', 'E-ROAD'], ['的', 'O'], ['监', 'B-TYPE'], ['控', 'E-TYPE']]" update_tag_scheme(dev_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist # print("map_file:{}".format(FLAGS.map_file)) # print("pre_emb:{}".format(FLAGS.pre_emb)) # map_file: maps.pkl # pre_emb: False if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping( train_sentences, FLAGS.lower)[0] # character -> count dict dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), 0, len(test_data))) # '3027 / 0 / 361 sentences in train / dev / test.' # print("batch_size:{}".format(FLAGS.batch_size)) # batch_size: 20 train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) # print("config_file:{}".format(FLAGS.config_file)) # config_file: config_file if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) log_path = os.path.join("log", FLAGS.log_file) # log_path:log/train.log logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data # print("steps_per_epoch:{}".format(steps_per_epoch)) # steps_per_epoch: 152 with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) # print("steps_check:{}".format(FLAGS.steps_check)) # steps_check: 100 if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger) export(model, sess, "ner", "export_model")
def train(): # load data sets # 句子集合 = [[句子1],[句子2],[句子3]],句子1 = [我 O,在 O,。。。] #<class 'list'>: [['海', 'O'], ['钓', 'O'], ['比', 'O'], ['赛', 'O'], ['地', 'O'], ['点', 'O'], ['在', 'O'], ['厦', 'B-LOC'], ['门', 'I-LOC'], ['与', 'O'], ['金', 'B-LOC'], ['门', 'I-LOC'], ['之', 'O'], ['间', 'O'], ['的', 'O'], ['海', 'O'], ['域', 'O'], ['。', 'O']] # train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) # dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) # test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) from xlnet_base.xlnet_data_utils import XLNetDataUtils sp_model = spm.SentencePieceProcessor() sp_model.Load('./chinese_xlnet_base_L-12_H-768_A-12/spiece.model') train_data = XLNetDataUtils(sp_model, batch_size=FLAGS.batch_size, entry="train") dev_data = XLNetDataUtils(sp_model, batch_size=FLAGS.batch_size, entry="dev") test_data = XLNetDataUtils(sp_model, batch_size=FLAGS.batch_size, entry="test") dev_batch = dev_data.iteration() def datapadding(data): alldatalist = [] datalist = data.data max_length = 64 for i in range(len(datalist)): tmpdatalist = [] token = datalist[i][0] segmentid = datalist[i][1] inputid = datalist[i][2] inputmask = datalist[i][3] labellist = datalist[i][4] #token label if len(labellist) < max_length: for i in range(max_length - len(token)): labellist.append(0) elif len(labellist) > max_length: tmplabellist = [] for i in range(max_length): tmplabellist.append(labellist[i]) labellist = tmplabellist #segmentid inputid inputmask if len(segmentid) < max_length: for i in range(max_length - len(segmentid)): segmentid.append(0) inputid.append(0) inputmask.append(0) elif len(segmentid) > max_length: tmpsegmentid = [] tmpinputid = [] tmpinputmask = [] for i in range(max_length): tmpsegmentid.append(segmentid[i]) tmpinputid.append(inputid[i]) tmpinputmask.append(inputmask[i]) segmentid = tmpsegmentid inputid = tmpinputid inputmask = tmpinputmask tmpdatalist.append(token) tmpdatalist.append(segmentid) tmpdatalist.append(inputid) tmpdatalist.append(inputmask) tmpdatalist.append(labellist) alldatalist.append(tmpdatalist) return alldatalist ftraindata = datapadding(train_data) fdevdata = datapadding(dev_data) ftestdata = datapadding(test_data) print(len(ftraindata)) print(len(fdevdata)) print(len(ftestdata)) # traindata = { # "batch_size": train_data.batch_size, # "input_size": train_data.input_size, # "vocab": train_data.vocab, # "tag_map": train_data.tag_map, # } # devdata = { # "batch_size": dev_data.batch_size, # "input_size": dev_data.input_size, # "vocab": dev_data.vocab, # "tag_map": dev_data.tag_map, # } # testdata = { # "batch_size": test_data.batch_size, # "input_size": test_data.input_size, # "vocab": test_data.vocab, # "tag_map": test_data.tag_map, # } # if not os.path.exists("./model/train_data_map.pkl"): # f = open("./model/train_data_map.pkl", "wb") # pickle.dump(traindata, f) # f.close() # if not os.path.exists("./model/dev_data_map.pkl"): # f = open("./model/dev_data_map.pkl", "wb") # pickle.dump(devdata, f) # f.close() # if not os.path.exists("./model/test_data_map.pkl"): # f = open("./model/test_data_map.pkl", "wb") # pickle.dump(testdata, f) # f.close() # Use selected tagging scheme (IOB / IOBES) #update_tag_scheme(train_sentences, FLAGS.tag_schema) #update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # Create a dictionary and a mapping for tags ''' _t:{'O': 869087, 'B-LOC': 16571, 'I-LOC': 22531, 'B-PER': 8144, 'I-PER': 15881, 'B-ORG': 9277, 'I-ORG': 37689, '[SEP]': 8, '[CLS]': 10} id_to_tag:{0: 'O', 1: 'I-ORG', 2: 'I-LOC', 3: 'B-LOC', 4: 'I-PER', 5: 'B-ORG', 6: 'B-PER', 7: '[CLS]', 8: '[SEP]'} tag_to_id:{'O': 0, 'I-ORG': 1, 'I-LOC': 2, 'B-LOC': 3, 'I-PER': 4, 'B-ORG': 5, 'B-PER': 6, '[CLS]': 7, '[SEP]': 8} ''' tag_to_id = train_data.tag_map id_to_tag = {v: k for k, v in tag_to_id.items()} with open(FLAGS.map_file, "wb") as f: pickle.dump([tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index ''' [['在', '这', '里', '恕', '弟', '不', '恭', '之', '罪', ',', '敢', '在', '尊', '前', '一', '诤', ':', '前', '人', '论', '书', ',', '每', '曰', '“', '字', '字', '有', '来', '历', ',', '笔', '笔', '有', '出', '处', '”', ',', '细', '读', '公', '字', ',', '何', '尝', '跳', '出', '前', '人', '藩', '篱', ',', '自', '隶', '变', '而', '后', ',', '直', '至', '明', '季', ',', '兄', '有', '何', '新', '出', '?'], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [101, 1762, 6821, 7027, 2609, 2475, 679, 2621, 722, 5389, 8024, 3140, 1762, 2203, 1184, 671, 6420, 8038, 1184, 782, 6389, 741, 8024, 3680, 3288, 100, 2099, 2099, 3300, 3341, 1325, 8024, 5011, 5011, 3300, 1139, 1905, 100, 8024, 5301, 6438, 1062, 2099, 8024, 862, 2214, 6663, 1139, 1184, 782, 5974, 5075, 8024, 5632, 7405, 1359, 5445, 1400, 8024, 4684, 5635, 3209, 2108, 8024, 1040, 3300, 862, 3173, 1139, 8043, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] ''' # train_data = prepare_dataset( # train_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower # ) # dev_data = prepare_dataset( # dev_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower # ) # test_data = prepare_dataset( # test_sentences, FLAGS.max_seq_len, tag_to_id, FLAGS.lower # ) print("%i / %i / %i sentences in train / dev / test." % (len(train_data.data), len(dev_data.data), len(test_data.data))) train_manager = BatchManager(ftraindata, FLAGS.batch_size) dev_manager = BatchManager(fdevdata, FLAGS.batch_size) test_manager = BatchManager(ftestdata, FLAGS.batch_size) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, config, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger, global_steps=step) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def main() -> None: """ Program entry point. Parses command line arguments to decide which dataset and model to use. :return: None. """ parse_command_line_arguments() print_num_gpus_available() gpu = tf.config.experimental.list_physical_devices(device_type='GPU') tf.config.experimental.set_memory_growth(gpu[0], True) # Start recording time. start_time = time.time() # Create label encoder. l_e = create_label_encoder() # Multiclass classification (mini-MIAS dataset) if config.dataset == "mini-MIAS": # Import entire dataset. images, chars, labels = import_minimias_dataset( data_dir="../data/{}/images".format(config.dataset), label_encoder=l_e) # Split dataset into training/test/validation sets (60%/20%/20% split). X_train, X_test, y_train, y_test = dataset_stratified_split( split=0.20, dataset=images, labels=labels) X_train, X_val, y_train, y_val = dataset_stratified_split( split=0.25, dataset=X_train, labels=y_train) if config.SAMPLING == 'x': X_train_rebalanced = X_train y_train_rebalanced = y_train else: print(len(y_train)) print(l_e.classes_) print(y_train.sum(axis=0)) if len(config.CLASS_TYPE.split('-')) == 2: if config.SAMPLING == 'up': X_train_rebalanced, y_train_rebalanced = generate_image_transforms_upsample( X_train, y_train) elif config.SAMPLING == 'down': X_train_rebalanced, y_train_rebalanced = generate_image_transforms_downsample( X_train, y_train) if len(config.CLASS_TYPE.split( '-')) != 2 and config.SAMPLING == 'up': X_train_rebalanced, y_train_rebalanced = generate_image_transforms( X_train, y_train) print(len(y_train_rebalanced)) print(l_e.classes_) print(y_train_rebalanced.sum(axis=0)) # Create and train CNN model. if config.cnn == "ResNet": if config.model == 'basic': model = generate_resnet_model(l_e.classes_.size) elif config.cnn == "VGG": if config.model == 'basic': model = generate_vgg_model(l_e.classes_.size) else: model = generate_vgg_model_advance(l_e.classes_.size) if config.run_mode == "train": model = train_network(l_e.classes_.size, model, X_train_rebalanced, y_train_rebalanced, X_val, y_val, config.BATCH_SIZE, config.EPOCH_1, config.EPOCH_2) # Binary classification (CBIS-DDSM dataset). elif config.dataset == "CBIS-DDSM": images, labels, density, cc, mlo = import_cbisddsm_training_dataset( l_e) images_test, labels_test, density_test, cc_test, mlo_test = import_cbisddsm_testing_dataset( l_e) if len(config.model.split('-')) > 1 and config.model.split( '-')[1] == '3': if config.run_mode == "train": X = np.vstack((images, density, cc, mlo)) elif config.run_mode == "test": X_test = np.vstack( (images_test, density_test, cc_test, mlo_test)) X_test = X_test.transpose() else: if config.run_mode == "train": X = np.vstack((images, density)) elif config.run_mode == "test": X_test = np.vstack((images_test, density_test)) X_test = X_test.transpose() if config.run_mode == "test": y_test = labels_test if config.run_mode == "train": # Split training dataset into training/validation sets (75%/25% split). X_train, X_val, y_train, y_val = dataset_stratified_split( split=0.25, dataset=X.transpose(), labels=labels) # X_train, X_val, y_train, y_val = dataset_stratified_split(split=0.25, dataset=images, labels=labels) if config.run_mode == "train": dataset_train = create_dataset(X_train, y_train) dataset_val = create_dataset(X_val, y_val) elif config.run_mode == "test": dataset_test = create_dataset(X_test, y_test) # Create and train CNN model. if config.cnn == "ResNet": if config.model.startswith('basic'): if len(config.model.split('-')) == 1: model = generate_resnet_model(l_e.classes_.size) else: model = generate_resnet_model_and_density( l_e.classes_.size) else: if len(config.model.split('-')) == 1: model = generate_resnet_model_advance(l_e.classes_.size) else: model = generate_resnet_model_advance_and_density( l_e.classes_.size) elif config.cnn == "VGG": if config.model.startswith('basic'): if len(config.model.split('-')) == 1: model = generate_vgg_model(l_e.classes_.size) else: model = generate_vgg_model_and_density(l_e.classes_.size) else: if len(config.model.split('-')) == 1: model = generate_vgg_model_advance(l_e.classes_.size) else: model = generate_vgg_model_advance_and_density( l_e.classes_.size) if config.run_mode == "train": model = train_network(l_e.classes_.size, model, dataset_train, None, dataset_val, None, config.BATCH_SIZE, config.EPOCH_1, config.EPOCH_2) else: print_error_message() try: # Save the model # model.save("../saved_models/dataset-{}_model-{}-{}_" + datetime.now().strftime("%d%Y%H%M%S") + ".h5".format(config.dataset, config.model, config.cnn)) if config.run_mode == "train": save_time = datetime.now().strftime("%Y%m%d%H%M") model.save_weights( "/cs/tmp/sjc29/saved_models/dataset-{}_model-{}-{}_{}.h5". format(config.dataset, config.model, config.cnn, save_time)) print('save_time: ', save_time) print_runtime("Finish Training", round(time.time() - start_time, 2)) elif config.run_mode == "test": model.load_weights( "/cs/tmp/sjc29/saved_models/dataset-{}_model-{}-{}_{}.h5". format(config.dataset, config.model, config.cnn, config.MODEL_SAVE_TIME)) except: print('save/load model error: ' + sys.exc_info()[0]) # print config print_config() # Evaluate model results. if config.dataset == "mini-MIAS": if config.run_mode == "train": y_pred = make_predictions(model, X_val) evaluate(y_val, y_pred, l_e, config.dataset, config.CLASS_TYPE, 'output') print_runtime("Finish Prediction Validation Set", round(time.time() - start_time, 2)) elif config.run_mode == "test": y_pred_test = make_predictions(model, X_test) evaluate(y_test, y_pred_test, l_e, config.dataset, config.CLASS_TYPE, 'output_test') print_runtime("Finish Prediction Testing Set", round(time.time() - start_time, 2)) elif config.dataset == "CBIS-DDSM": if config.run_mode == "train": y_pred = make_predictions(model, dataset_val) evaluate(y_val, y_pred, l_e, config.dataset, config.CLASS_TYPE, 'output') print_runtime("Finish Prediction Validation Set", round(time.time() - start_time, 2)) elif config.run_mode == "test": y_pred_test = make_predictions(model, dataset_test) evaluate(y_test, y_pred_test, l_e, config.dataset, config.CLASS_TYPE, 'output_test') print_runtime("Finish Prediction Testing Set", round(time.time() - start_time, 2)) # Print the prediction # print(y_pred) # Print training runtime. print_runtime("Total", round(time.time() - start_time, 2))
# Delay time in seconds for API call returns in case of auth # failures (some DoS-countermeasure). For testing purposes we # usually want this as fast as possible. This value gets used in # create_mq_objects.py for the creation of the queue manager conn # auth. FAIL_DELAY = os.environ.get( 'PYMQI_TEST_QM_CONN_AUTH_FAIL_DELAY', '0') # Queue naming setup class QUEUE: PREFIX = os.environ.get('PYMQI_TEST_QUEUE_PREFIX', '') QUEUE_NAMES = { 'TestRFH2PutGet': PREFIX + 'TEST.PYMQI.RFH2PUTGET', 'TestQueueManager': PREFIX + 'TEST.PYMQI.QUEUEMANAGER', } # convenience attribute derived from above settings, may be used for tests # that mandate the MQSERVER environment variable # E.g. MQSERVER="SVRCONN.1/TCP/mq.example.org(1777)" MQSERVER = '%(channel)s/%(transport)s/%(host)s(%(port)s)' % { 'channel': QM.CHANNEL, 'transport': QM.TRANSPORT, 'host': QM.HOST, 'port': QM.PORT, } if __name__ == '__main__': utils.print_config(PATHS, MQ)
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist, load data if exists maps if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # 设置训练日志目录 train_log = os.path.join(FLAGS.logdir, "train") if not os.path.exists(train_log): os.makedirs(train_log) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data # the nums of batch data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) # 观察所建立的计算图 train_writer = tf.summary.FileWriter(train_log, sess.graph) logger.info("start training") loss = [] dev_f1 = [] test_f1 = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss, merged = model.run_step( sess, True, batch) # step是global step # 在迭代中输出到结果 train_writer.add_summary(merged, step) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] # use dev data to validation the model best, dev_f1_value = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) # store the dev f1 dev_f1.append(dev_f1_value) if best: save_model(sess, model, FLAGS.ckpt_path, logger) # use current the model to test _, test_f1_value = evaluate(sess, model, "test", test_manager, id_to_tag, logger) # store the test f1 test_f1.append(test_f1_value) # write the dev_f1 and test_f1 to file f1_result = {} f1_result["dev_f1"] = dev_f1 f1_result["test_f1"] = test_f1 write_data_to_file(f1_result, "f1_result")
def train(): # load data sets train_sentences = load_sentences( FLAGS.train_file, FLAGS.lower, FLAGS.zeros) # dimension:num_sentence*len_sentence*2 dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme( train_sentences, FLAGS.tag_schema) # dimension:num_sentence*len_sentence*2 update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: # 如果使用预训练的词嵌入 dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[ 0] # dico_chars_train dimension: 训练数据集中出现的字符类别数*2, dico_chars, char_to_id, id_to_char = augment_with_pretrained( # 利用测试数据样本集中的字对dico_chars_train进行补充 dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: # 创建map_file文件 pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset( train_sentences, char_to_id, tag_to_id, FLAGS.lower) # dimension: NumSentence*4*LenSentence dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) train_manager = BatchManager( train_data, FLAGS.batch_size ) # batch_data dimension: BatchNum*4*BatchSize*MaxLenSentence dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): # 若已有config_file则读取加载 config = load_config(FLAGS.config_file) else: # 若没有config_file则新建并保存为文件 config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # 将config打印到日志文件 # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # 动态申请内存 steps_per_epoch = train_manager.len_data # len_data: ceil(NumSentence/BatchSize) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(FLAGS.max_epoch): # 括号中数字是epoach数量 for batch in train_manager.iter_batch( shuffle=True ): # 一次从batch_data中取出一个batch,Shuffle为True表示打乱batch_data的顺序 step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger) # View the tensorboard graph by running the following code and then going to the terminal and typing: # tensorboard --logdir = tensorboard_logs merged = tf.summary.merge_all() if not os.path.exists('tensorboard_logs/'): os.makedirs('tensorboard_logs/') my_writer = tf.summary.FileWriter('tensorboard_logs/', sess.graph)
def train(): train_sentences, dico, char_to_id, id_to_char = load_sentence( FLAGS.train_file) if not os.path.isfile(FLAGS.map_file): if FLAGS.pre_emb: dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico.copy(), FLAGS.emb_file, ) else: sentences, dico, char_to_id, id_to_char = load_sentence( FLAGS.train_file) print(train_sentences[0]) with open(FLAGS.map_file, 'wb') as f: pickle.dump([char_to_id, id_to_char], f) else: with open(FLAGS.map_file, 'rb') as f: char_to_id, id_to_char = pickle.load(f) train_data, test_data, dev_data = prepare_dataset(train_sentences, char_to_id) print(train_data[0]) print(test_data[0]) print(dev_data[0]) print(len(train_data), len(dev_data), len(test_data)) train_manager = BatchManager(train_data, FLAGS.batch_size) test_manager = BatchManager(test_data, 100) dev_manager = BatchManager(dev_data, 100) make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3" gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) tf_config = tf.ConfigProto(gpu_options=gpu_options) tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] best = 0 # sess.graph.finalize() for i in range(50): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{},".format( iteration, step % steps_per_epoch, steps_per_epoch)) loss = [] Acc_result = evaluate(sess, model, "dev", dev_manager, logger) logger.info("Acc{}".format(Acc_result)) logger.info("test") # precision, recall, f1_score = model.evaluete_(sess,test_manager) # logger.info("P, R, F,{},{},{}".format(precision, recall, f1_score)) test_result = evaluate(sess, model, "test", test_manager, logger) if test_result > best: best = test_result save_model(sess, model, FLAGS.ckpt_path, logger)
dataset_masks_name = config.get('Paths', 'dataset_masks_name') annotations_file_name = config.get('Paths', 'annotations_file_name') final_annotations_file_name = config.get('Paths', 'final_annotations_file_name') type_of_border = config.get('Annotations', 'type_of_border') patch_size_height = config.getint('MultiScale', 'patch_size_height') patch_size_width = config.getint('MultiScale', 'patch_size_width') flip = config.getboolean('DataAugmentation', 'flip') rotate = config.getboolean('DataAugmentation', 'rotate') contrast = config.getboolean('DataAugmentation', 'contrast') if display_logs: print_config(config_path) # Build the new dataset create_dataset_hierarchy(path_dataset_created, augmented_dataset_name, dataset_images_name, dataset_masks_name) # Transfer original images transfer_original_images(path_dataset_created, augmented_dataset_name, dataset_name, dataset_images_name, dataset_masks_name) # Load the transformations if flip: data_augmentation_flip(path_dataset_created, augmented_dataset_name, dataset_images_name, dataset_masks_name)
def train(X_train,X_dev,X_test): # load data sets train_sentences = X_train dev_sentences = X_dev test_sentences = X_test train_sentences_loc = load_sentences(FLAGS.train_file_loc, FLAGS.lower, FLAGS.zeros) dev_sentences_loc = load_sentences(FLAGS.dev_file_loc, FLAGS.lower, FLAGS.zeros) test_sentences_loc = load_sentences(FLAGS.test_file_loc, FLAGS.lower, FLAGS.zeros) train_sentences_org = load_sentences(FLAGS.train_file_org, FLAGS.lower, FLAGS.zeros) dev_sentences_org = load_sentences(FLAGS.dev_file_org, FLAGS.lower, FLAGS.zeros) test_sentences_org = load_sentences(FLAGS.test_file_org, FLAGS.lower, FLAGS.zeros) train_sentences_per = load_sentences(FLAGS.train_file_per, FLAGS.lower, FLAGS.zeros) dev_sentences_per = load_sentences(FLAGS.dev_file_per, FLAGS.lower, FLAGS.zeros) test_sentences_per = load_sentences(FLAGS.test_file_per, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) update_tag_scheme(train_sentences_loc, FLAGS.tag_schema) update_tag_scheme(test_sentences_loc, FLAGS.tag_schema) update_tag_scheme(train_sentences_per, FLAGS.tag_schema) update_tag_scheme(test_sentences_per, FLAGS.tag_schema) update_tag_scheme(train_sentences_org, FLAGS.tag_schema) update_tag_scheme(test_sentences_org, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences]) ) ) dico_chars_train_loc = char_mapping(train_sentences_loc, FLAGS.lower)[0] dico_chars_loc, char_to_id_loc, id_to_char_loc = augment_with_pretrained( dico_chars_train_loc.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences_loc]) ) ) dico_chars_train_per = char_mapping(train_sentences_per, FLAGS.lower)[0] dico_chars_per, char_to_id_per, id_to_char_per = augment_with_pretrained( dico_chars_train_per.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences_per]) ) ) dico_chars_train_org = char_mapping(train_sentences_org, FLAGS.lower)[0] dico_chars_org, char_to_id_org, id_to_char_org = augment_with_pretrained( dico_chars_train_org.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences_org]) ) ) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) _c_loc, char_to_id_loc, id_to_char_loc = char_mapping(train_sentences_loc, FLAGS.lower) _c_per, char_to_id_per, id_to_char_per = char_mapping(train_sentences_per, FLAGS.lower) _c_org, char_to_id_org, id_to_char_org = char_mapping(train_sentences_org, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) _t_loc, tag_to_id_loc, id_to_tag_loc = tag_mapping(train_sentences_loc) _t_per, tag_to_id_per, id_to_tag_per = tag_mapping(train_sentences_per) _t_org, tag_to_id_org, id_to_tag_org = tag_mapping(train_sentences_org) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag,char_to_id_loc, id_to_char_loc, tag_to_id_loc, id_to_tag_loc,char_to_id_per, id_to_char_per, tag_to_id_per, id_to_tag_per,char_to_id_org, id_to_char_org, tag_to_id_org, id_to_tag_org], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag,char_to_id_loc, id_to_char_loc, tag_to_id_loc, id_to_tag_loc,char_to_id_per, id_to_char_per, tag_to_id_per, id_to_tag_per,char_to_id_org, id_to_char_org, tag_to_id_org, id_to_tag_org = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset( train_sentences, char_to_id, tag_to_id, FLAGS.lower ) dev_data = prepare_dataset( dev_sentences, char_to_id, tag_to_id, FLAGS.lower ) test_data = prepare_dataset( test_sentences, char_to_id, tag_to_id, FLAGS.lower ) print("%i / %i / %i sentences in train / dev / test." % ( len(train_data),len(dev_data), len(test_data))) train_data_loc = prepare_dataset_ner( train_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower ) dev_data_loc = prepare_dataset_ner( dev_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower ) test_data_loc = prepare_dataset_ner( test_sentences_loc, char_to_id_loc, tag_to_id_loc, FLAGS.lower ) print("%i / %i / %i sentences_loc in train / dev / test." % ( len(train_data_loc), len(dev_data_loc), len(test_data_loc))) train_data_per = prepare_dataset_ner( train_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower ) dev_data_per = prepare_dataset_ner( dev_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower ) test_data_per = prepare_dataset_ner( test_sentences_per, char_to_id_per, tag_to_id_per, FLAGS.lower ) print("%i / %i / %i sentences_per in train / dev / test." % ( len(train_data_per), len(dev_data_per), len(test_data_per))) train_data_org = prepare_dataset_ner( train_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower ) dev_data_org = prepare_dataset_ner( dev_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower ) test_data_org = prepare_dataset_ner( test_sentences_org, char_to_id_org, tag_to_id_org, FLAGS.lower ) print("%i / %i / %i sentences_org in train / dev / test." % ( len(train_data_org), len(dev_data_org), len(test_data_org))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) train_manager_loc = BatchManager(train_data_loc, FLAGS.batch_size) train_manager_per = BatchManager(train_data_per, FLAGS.batch_size) train_manager_org = BatchManager(train_data_org, FLAGS.batch_size) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id,char_to_id_loc, tag_to_id_loc,char_to_id_per, tag_to_id_per,char_to_id_org, tag_to_id_org) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data steps_per_epoch_loc = train_manager_loc.len_data steps_per_epoch_per = train_manager_per.len_data steps_per_epoch_org = train_manager_org.len_data model = create_model(Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, id_to_char_loc, id_to_char_per, id_to_char_org, logger) with tf.Session(config=tf_config, graph = model.graph ) as sess: sess.run(tf.global_variables_initializer()) if config["pre_emb"]: emb_weights = sess.run(model.char_lookup.read_value()) emb_weights_ner = sess.run(model.char_lookup.read_value()) emb_weights, emb_weights_ner = load_word2vec(config["emb_file"], id_to_char, id_to_char_loc,id_to_char_per,id_to_char_org, config["char_dim"], emb_weights, emb_weights_ner) sess.run(model.char_lookup.assign(emb_weights)) logger.info("Load pre-trained embedding.") logger.info("start training") loss = [] loss_loc = [] loss_per = [] loss_org = [] for i in range(100): for batch_loc in train_manager_loc.iter_batch(shuffle=True): step_loc, batch_loss_loc = model.run_step_ner(sess, True, batch_loc) loss_loc.append(batch_loss_loc) if step_loc % FLAGS.steps_check == 0: iteration_loc = step_loc // steps_per_epoch_loc + 1 logger.info("iteration:{} step_loc:{}/{}, " "NER loss:{:>9.6f}".format( iteration_loc, step_loc % steps_per_epoch_loc, steps_per_epoch_loc, np.mean(loss_loc))) loss_loc = [] for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration_1 = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "SKILL loss:{:>9.6f}".format( iteration_1, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] precision_loc_dev = model.precision(sess, dev_manager, id_to_tag) precision_loc_test = model.precision(sess, test_manager, id_to_tag) for batch_per in train_manager_per.iter_batch(shuffle=True): step_per, batch_loss_per = model.run_step_ner(sess, True, batch_per) loss_per.append(batch_loss_per) if step_per % FLAGS.steps_check == 0: iteration_per = step_per // steps_per_epoch_per + 1 logger.info("iteration:{} step_per:{}/{}, " "NER loss:{:>9.6f}".format( iteration_per, step_per % steps_per_epoch_per, steps_per_epoch_per, np.mean(loss_per))) loss_per = [] for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration_2 = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "SKILL loss:{:>9.6f}".format( iteration_2, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] precision_per_dev = model.precision(sess, dev_manager, id_to_tag) precision_per_test = model.precision(sess, test_manager, id_to_tag) for batch_org in train_manager_org.iter_batch(shuffle=True): step_org, batch_loss_org = model.run_step_ner(sess, True, batch_org) loss_org.append(batch_loss_org) if step_org % FLAGS.steps_check == 0: iteration_org = step_org // steps_per_epoch_org + 1 logger.info("iteration:{} step_org:{}/{}, " "NER loss:{:>9.6f}".format( iteration_org, step_org % steps_per_epoch_org, steps_per_epoch_org, np.mean(loss_org))) loss_org = [] for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration_3 = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "SKILL loss:{:>9.6f}".format( iteration_3, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] precision_org_dev = model.precision(sess, dev_manager, id_to_tag) precision_org_test = model.precision(sess, test_manager, id_to_tag) best = evaluate(sess, model, "dev", dev_manager, id_to_tag,precision_loc_dev,precision_per_dev,precision_org_dev, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger) best_test,results= evaluate(sess, model, "test", test_manager, id_to_tag,precision_loc_test,precision_per_test,precision_org_test, logger) with open("CDTL_PSE-result.csv", "a",encoding='utf-8')as st_re: st_re.write(str(results).replace("[", "").replace("]", "")) st_re.write("\n")
help='disable GPU training') args = parser.parse_args() args.bi = not args.no_bi args.cuda = torch.cuda.is_available() and not args.no_cuda if args.cuda: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" sorted_gpu_info = get_gpu_memory_map() for gpu_id, (mem_left, util) in sorted_gpu_info: if mem_left >= 1000: os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) # args.cuda = f'cuda:{gpu_id}' print('use gpu:{} with {} MB left, util {}%'.format(gpu_id, mem_left, util)) break else: print(f'no gpu has memory left >= {args.min_mem} MB, exiting...') exit() else: torch.set_num_threads(10) args.data = 'train010203x_test04' args.remark = 'del' remark = f'_{args.data}_{args.remark}' current_time = datetime.now().strftime('%b%d_%H-%M-%S') args.path = os.path.join('saved_model/debug_ext_ml', current_time + remark) print_config(args) os.environ["DATA"] = f"./data/DUC-json/{args.data}" DATA_DIR = os.environ['DATA'] main(args)
def train(): # load data sets # train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) # dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) all_train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) train_sentences, dev_sentences = split_train_dev(all_train_sentences) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # update_tag_scheme(dev_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: # dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars_train = char_mapping(all_train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(all_train_sentences, FLAGS.lower) # _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(all_train_sentences) # _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # nlp = StanfordCoreNLP(r'E:\DC\dataset\泰一指尚评测数据\stanford-corenlp-full-2017-06-09') #l_sorted_lexcion = load_lexcion(FLAGS.lexcion_file, nlp) l_sorted_lexcion = [] # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, l_sorted_lexcion, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, l_sorted_lexcion, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, l_sorted_lexcion, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) max_len = max( [len(sentence[0]) for sentence in train_data + test_data + dev_data]) train_manager = BatchManager(train_data, FLAGS.batch_size, max_len) dev_manager = BatchManager(dev_data, 800, max_len) test_manager = BatchManager(test_data, 800, max_len) # random.shuffle(train_data) # pad_test_data = pad_data(test_data) # pad_dev_data = pad_data(dev_data) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id, max_len) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(FLAGS.max_epoch): random.shuffle(train_data) pad_train_data = pad_data(train_data, max_len) strings, chars, lexcion_teatures, pos_ids, dep_ids, head_ids, targets = pad_train_data for j in range(0, len(strings), FLAGS.batch_size): batch = [ strings[j:j + FLAGS.batch_size], chars[j:j + FLAGS.batch_size], lexcion_teatures[j:j + FLAGS.batch_size], pos_ids[j:j + FLAGS.batch_size], dep_ids[j:j + FLAGS.batch_size], head_ids[j:j + FLAGS.batch_size], targets[j:j + FLAGS.batch_size] ] step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "AS loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger, i) evaluate(sess, model, "test", test_manager, id_to_tag, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] #best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) #if best: save_model(sess, model, FLAGS.ckpt_path, logger)
def train_new(): train_sent = load_sentences(FLAGS.filepath) update_tag_scheme(train_sent, FLAGS.tag_schema) if not os.path.isfile(FLAGS.map_file): _c, char_to_id, id_to_char = char_mapping(train_sent, FLAGS.lower) print("random embedding") # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sent) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # 数据准备,划分验证集和训练集 np.random.seed(10) train_sent_ = np.array(train_sent) shuffle_indices = np.random.permutation(np.arange(len(train_sent))) sent_shuffled = train_sent_[shuffle_indices] dev_sample_index = -1 * int(FLAGS.dev_percentage * float(len(train_sent))) train_sent_new, dev_sent = sent_shuffled[:dev_sample_index], sent_shuffled[ dev_sample_index:] train_data = prepare_dataset(train_sent_new, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sent, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i sentences in train." % (len(train_data), len(dev_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = FLAGS.log_file logger = get_logger(log_path) print_config(config, logger) # 根据需求,设置动态使用GPU资源 tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: fig = plt.figure() ax = fig.add_subplot(211) ax2 = fig.add_subplot(212) plt.grid(True) plt.ion() model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(FLAGS.max_epoch): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % 20 == 0: ax.scatter(step, np.mean(loss), c='b', marker='.') plt.pause(0.001) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best, f1 = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) ax2.scatter(i + 1, f1, c='b', marker='.') plt.pause(0.001) if best: save_model(sess, model, FLAGS.ckpt_path, logger, "best")