def print_config_summary(config, args): # Summarize config on the screen as a sanity check print(100 * "=") print(f"Here is the configuration being used:\n") print_config(config) print(100 * "=") print(f"Arguments being used:\n") print_config(args) print(100 * "=")
def main_train(): # load data sets # sentences = [[(words11, tag11), ...], [(word21, tag21), ...], ...] train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) # 更新在train_sentences和test_sentences中 update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist # 创建或加载字符、词、特征、target的映射字典 if not os.path.isfile(FLAGS.map_file): # create dictionary for word # 若存在pre-trained embedding file,则同时使用pre-trained和训练集构建字典 if FLAGS.pre_emb: # 统计train中字,返回字典 dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] # 使用pre-trained的字增大训练集字的字典 dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list(itertools.chain.from_iterable( [[w[0] for w in s] for s in test_sentences]) ) ) # 否则只是用训练集构建字典 else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, target_to_id, id_to_target = tag_mapping(train_sentences) # 创建其他特征的映射字典,返回的三个都为dict _f, feature_to_id, id_to_feature = feature_mapping(train_sentences, FLAGS.features) # 存储字、target、feature的映射关系 with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, target_to_id, id_to_target, feature_to_id, id_to_feature], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, target_to_id, id_to_target, feature_to_id, id_to_feature = pickle.load(f) # make path for store log and model config if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = create_config_model(FLAGS, char_to_id, target_to_id, feature_to_id) logger = get_logger(FLAGS.log_file) print_config(config, logger) train(config, train_sentences, dev_sentences, test_sentences, char_to_id, feature_to_id, target_to_id, id_to_char, id_to_target, logger)
parser.add_argument('--batch_size', type=int, default=3, help='Batch size') parser.add_argument('--gpu_device', type=str, default=0, help='GPU device (number) to use; defaults to 0') parser.add_argument('--cpu', action='store_true', help='Whether to use CPU instead of GPU; this option overwrites the --gpu_device argument') parser.add_argument('--save_scores', action='store_true', help='Whether to save model scores during inference') parser.add_argument('--split', default='test', help='Dataset split; possible values are \'training\', \'validation\', \'test\'') args = parser.parse_args() with open(os.path.join(args.config_path, 'config.pkl'), 'rb') as f: cfg_dict = pickle.load(f) if 'merge_function' not in cfg_dict: cfg_dict['merge_function'] = 'concat' if 'zero_shot' not in cfg_dict: cfg_dict['zero_shot'] = False cfg_dict['classes_to_exclude'] = None if 'num_features_mixed5c' not in cfg_dict: cfg_dict['num_features_mixed5c'] = 1024 #if 'i3d_weights_path' not in cfg_dict: # cfg_dict['i3d_weights_path'] = 'models/' cfg_dict['i3d_weights_path'] = 'models/' if 'class_map' not in cfg_dict: cfg_dict['class_map'] = utils.class2idx_map(cfg_dict['classes_to_exclude']) cfg = config.GetConfig(**cfg_dict) utils.print_config(cfg) dataloader, test_set, model, device = prepare_inference(cfg, args) inference(dataloader, test_set, model, device, cfg)
import matplotlib.pyplot as plt from collections import OrderedDict import h5py from utils.utils import prepare_output_dirs, print_config, write_config from utils.feature_extraction import read_data from utils.model import create_model from utils.visdata import save_history, plot_confusion_matrix from utils.SGDW import SGDW from config import parse_opts config = parse_opts() config = prepare_output_dirs(config) print_config(config) write_config(config, os.path.join(config.save_dir, 'config.json')) os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = config.device #################################################################### #################################################################### # Parameters song_samples = 660000 genres = { 'metal': 0, 'disco': 1, 'classical': 2, 'hiphop': 3,
def train_ner(): clean(FLAGS) # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) update_tag_scheme(train_sentences, FLAGS.tag_schema) update_tag_scheme(test_sentences, FLAGS.tag_schema) update_tag_scheme(dev_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word if FLAGS.pre_emb: dico_chars_train = char_mapping(train_sentences, FLAGS.lower)[0] dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), FLAGS.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), 0, len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join("log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss = [] for i in range(25): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, FLAGS.ckpt_path, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train(): # load data sets train_sentences = load_sentences(args.train_file, args.lower, args.zeros) dev_sentences = load_sentences(args.dev_file, args.lower, args.zeros) test_sentences = load_sentences(args.test_file, args.lower, args.zeros) # Use selected tagging scheme (IOB / IOBES) # 检测并维护数据集的 tag 标记 update_tag_scheme(train_sentences, args.tag_schema) update_tag_scheme(test_sentences, args.tag_schema) update_tag_scheme(dev_sentences, args.tag_schema) # create maps if not exist # 根据数据集创建 char_to_id, id_to_char, tag_to_id, id_to_tag 字典,并储存为 pkl 文件 if not os.path.isfile(args.map_file): # create dictionary for word if args.pre_emb: dico_chars_train = char_mapping(train_sentences, args.lower)[0] # 利用预训练嵌入集增强(扩充)字符字典,然后返回字符与位置映射关系 dico_chars, char_to_id, id_to_char = augment_with_pretrained( dico_chars_train.copy(), args.emb_file, list( itertools.chain.from_iterable([[w[0] for w in s] for s in test_sentences]))) else: _c, char_to_id, id_to_char = char_mapping(train_sentences, args.lower) # Create a dictionary and a mapping for tags # 获取标记与位置映射关系 tag_to_id, id_to_tag, intent_to_id, id_to_intent = tag_mapping( train_sentences) with open(args.map_file, "wb") as f: pickle.dump([ char_to_id, id_to_char, tag_to_id, id_to_tag, intent_to_id, id_to_intent ], f) else: with open(args.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag, intent_to_id, id_to_intent = pickle.load( f) # 提取句子特征 # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, char_to_id, tag_to_id, intent_to_id, args.lower) dev_data = prepare_dataset(dev_sentences, char_to_id, tag_to_id, intent_to_id, args.lower) test_data = prepare_dataset(test_sentences, char_to_id, tag_to_id, intent_to_id, args.lower) # code.interact(local=locals()) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) # 获取可供模型训练的单个批次数据 train_manager = BatchManager(train_data, args.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) # make path for store log and model if not exist make_path(args) if os.path.isfile(args.config_file): config = load_config(args.config_file) else: config = config_model(char_to_id, tag_to_id, intent_to_id) save_config(config, args.config_file) make_path(args) logger = get_logger(args.log_file) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # 训练集全量跑一次需要迭代的次数 steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: # 此处模型创建为项目最核心代码 model = create_model(sess, Model, args.ckpt_path, load_word2vec, config, id_to_char, logger) logger.info("start training") loss_slot = [] loss_intent = [] # with tf.device("/gpu:0"): for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss_slot, batch_loss_intent = model.run_step( sess, True, batch) loss_slot.append(batch_loss_slot) loss_intent.append(batch_loss_intent) if step % args.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "INTENT loss:{:>9.6f}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss_intent), np.mean(loss_slot))) loss_slot = [] loss_intent = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: # if i%7 == 0: save_model(sess, model, args.ckpt_path, logger) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def prepare_training(cfg, args): if cfg.use_gpu: os.environ['CUDA_VISIBLE_DEVICES'] = cfg.device_list if cfg.use_gpu and torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') if cfg.model_name == 'baseline': # initalize baseline model model = baseline_model.BaseNet(cfg) elif cfg.model_name == 'gcn': # initalize gcn model model = gcn_model.GCNNet(cfg) else: assert ( False ), 'Variable model_name should be either \'baseline\' or 2 \'gcn\'.' model = model.to(device=device) optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=1, momentum=cfg.momentum, weight_decay=cfg.weight_decay) if cfg.resume_training: cfg.filename = 'checkpoint_' + cfg.checkpoint_path.split('/')[-2] model, optimizer, start_epoch = load_checkpoint( model, optimizer, cfg.checkpoint_path) cfg.start_epoch = start_epoch else: cfg.filename = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) utils.make_dirs( os.path.join(cfg.results_path, cfg.model_name, cfg.filename)) utils.save_config(cfg) utils.print_config(cfg) if cfg.save_scores: utils.make_dirs( os.path.join(cfg.scores_path, cfg.model_name, cfg.filename, datasets['train'].split)) utils.make_dirs( os.path.join(cfg.scores_path, cfg.model_name, cfg.filename, datasets['val'].split)) if cfg.plot_grad_flow: utils.make_dirs( os.path.join(cfg.results_path, cfg.model_name, cfg.filename, 'grad_flow')) #utils.save_config(cfg) #utils.print_config(cfg) training_set = dataset.return_dataset(args.data_path, args.annot_path, cfg, 'training') validation_set = dataset.return_dataset(args.data_path, args.annot_path, cfg, 'validation') datasets = {'train': training_set, 'val': validation_set} training_loader = data.DataLoader(training_set, batch_size=cfg.training_batch_size, shuffle=True, num_workers=4, worker_init_fn=worker_init_fn) validation_loader = data.DataLoader(validation_set, batch_size=cfg.validation_batch_size, shuffle=False, num_workers=4) dataloaders = {'train': training_loader, 'val': validation_loader} return dataloaders, datasets, model, device, optimizer
def run_gdl(cfg: DictConfig) -> None: """ Function general for Geo Deep-Learning using Hydra library to rules all the parameters and functions using during the task wanted to execute. Process ------- 1. Read and convert the `gdl_config.yaml` to a dictionary. 2. Verify if the code and other information need to be save. 3. If the debug option is activate, the entire config yaml will be printed and save in a log. In addition of that, if the mode is `train`, a validation run will precede the training to assure the well functioning of the code. 4. Verify is the chosen mode is available. 5. Verify is the chosen task is available and run the code link to that task. ------- :param cfg: (DictConfig) Parameters and functions in the main yaml config file. """ cfg = OmegaConf.create(cfg) # debug config if cfg.debug: pass # cfg.training.num_sanity_val_steps = 1 # only work with pytorch lightning # logging.info(OmegaConf.to_yaml(cfg, resolve=True)) # check if the mode is chosen if type(cfg.mode) is DictConfig: msg = "You need to choose between those modes: {}" raise logging.critical(msg.format(list(cfg.mode.keys()))) # save all overwritten parameters logging.info('\nOverwritten parameters in the config: \n' + cfg.general.config_override_dirname) # Start ----------------------------------- msg = "Let's start {} for {} !!!".format(cfg.mode, cfg.task.task_name) logging.info("\n" + "-" * len(msg) + "\n" + msg + "\n" + "-" * len(msg)) # ----------------------------------------- # Start the timer start_time = time.time() # Read the task and execute it task = load_obj(cfg.task.path_task_function) task(cfg) # Add git hash from current commit to parameters. with open_dict(cfg): cfg.general.git_hash = get_git_hash() # Pretty print config using Rich library if cfg.get("print_config"): print_config(cfg, resolve=True) # End -------------------------------- msg = "End of {} !!!".format(cfg.mode) logging.info("\n" + "-" * len(msg) + "\n" + msg + "\n" + "Elapsed time: {:.2f}s".format(time.time() - start_time) + "\n" + "-" * len(msg) + "\n")
def train(): tf.io.gfile.mkdir(FLAGS.output) log_path = os.path.join(FLAGS.output, 'model.log') logger = get_logger(log_path) # load data sets train_sentences = load_sentences(os.path.join(FLAGS.data, "train.txt"), FLAGS.zeros) dev_sentences = load_sentences(os.path.join(FLAGS.data, "dev.txt"), FLAGS.zeros) test_sentences = load_sentences(os.path.join(FLAGS.data, "test.txt"), FLAGS.zeros) # create maps if not exist map_file = os.path.join(FLAGS.output, 'maps.pkl') if not os.path.isfile(map_file): # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) with open(map_file, "wb") as f: pickle.dump([tag_to_id, id_to_tag], f) else: with open(map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_dataset(train_sentences, FLAGS.max_seq_len, tag_to_id) dev_data = prepare_dataset(dev_sentences, FLAGS.max_seq_len, tag_to_id) test_data = prepare_dataset(test_sentences, FLAGS.max_seq_len, tag_to_id) logger.info("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, FLAGS.batch_size) test_manager = BatchManager(test_data, FLAGS.batch_size) # make path for store log and model if not exist config_file = os.path.join(FLAGS.output, 'config.json') if os.path.isfile(config_file): config = load_config(config_file) else: config = config_model(tag_to_id) save_config(config, config_file) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, os.path.join(FLAGS.output, 'checkpoint'), config, logger) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] best = evaluate(sess, model, "dev", dev_manager, id_to_tag, logger) if best: save_model(sess, model, os.path.join(FLAGS.output, 'checkpoint'), logger, global_steps=step) evaluate(sess, model, "test", test_manager, id_to_tag, logger)
def train(): # load data sets train_sentences = load_sentences(FLAGS.train_file, FLAGS.lower, FLAGS.zeros) dev_sentences = load_sentences(FLAGS.dev_file, FLAGS.lower, FLAGS.zeros) test_sentences = load_sentences(FLAGS.test_file, FLAGS.lower, FLAGS.zeros) # Use selected tagging scheme (IOB / IOBES) #update_tag_scheme(train_sentences, FLAGS.tag_schema) #update_tag_scheme(test_sentences, FLAGS.tag_schema) # create maps if not exist if not os.path.isfile(FLAGS.map_file): # create dictionary for word _c, char_to_id, id_to_char = char_mapping(train_sentences, FLAGS.lower) # Create a dictionary and a mapping for tags _t, tag_to_id, id_to_tag = tag_mapping(train_sentences) os.makedirs('%s' % FLAGS.save_path) with open(FLAGS.map_file, "wb") as f: pickle.dump([char_to_id, id_to_char, tag_to_id, id_to_tag], f) else: with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # prepare data, get a collection of list containing index train_data = prepare_padding_dataset(train_sentences, FLAGS.max_seq_len, char_to_id, tag_to_id, FLAGS.lower) dev_data = prepare_padding_dataset(dev_sentences, FLAGS.max_seq_len, char_to_id, tag_to_id, FLAGS.lower) test_data = prepare_padding_dataset(test_sentences, FLAGS.max_seq_len, char_to_id, tag_to_id, FLAGS.lower) print("%i / %i / %i sentences in train / dev / test." % (len(train_data), len(dev_data), len(test_data))) train_manager = BatchManager(train_data, FLAGS.batch_size) dev_manager = BatchManager(dev_data, 100) test_manager = BatchManager(test_data, 100) """ batch = train_manager.batch_data[0] strings, chars, segs, tags = batch for chrs in chars: print(chrs) for chrs in segs: print(chrs) print(tag_to_id) """ # make path for store log and model if not exist make_path(FLAGS) if os.path.isfile(FLAGS.config_file): config = load_config(FLAGS.config_file) else: config = config_model(char_to_id, tag_to_id) save_config(config, FLAGS.config_file) make_path(FLAGS) log_path = os.path.join(FLAGS.save_path, "log", FLAGS.log_file) logger = get_logger(log_path) print_config(config, logger) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True steps_per_epoch = train_manager.len_data with tf.Session(config=tf_config) as sess: model = TransformerCRFModel(config, is_training=True) sess.run(tf.global_variables_initializer()) logger.info("start training") loss = [] for i in range(100): for batch in train_manager.iter_batch(shuffle=True): step, batch_loss = model.run_step(sess, True, batch) loss.append(batch_loss) if step % FLAGS.steps_check == 0: iteration = step // steps_per_epoch + 1 logger.info("iteration:{} step:{}/{}, " "NER loss:{:>9.6f}".format( iteration, step % steps_per_epoch, steps_per_epoch, np.mean(loss))) loss = [] predict_lists = [] source_tag = [] best_dev_f1 = 0.0 best_test_f1 = 0.0 for batch in dev_manager.iter_batch(shuffle=False): lengths, logits = model.run_step(sess, False, batch) _, chars, segs, tags = batch transition = model.transition.eval(session=sess) pre_seq = model.predict(logits, transition, lengths) pre_label = recover_label(pre_seq, lengths, id_to_tag) """ for p in range(len(pre_label)): print(chars[p]) print(pre_label[p]) """ source_label = recover_label(tags, lengths, id_to_tag) predict_lists.extend(pre_label) source_tag.extend(source_label) train_loss_v = np.round(float(np.mean(loss)), 4) print('****************************************************') acc, p, r, f = get_ner_fmeasure(source_tag, predict_lists, config["tag_schema"]) logger.info('epoch:\t{}\ttrain loss:\t{}\t'.format( i + 1, train_loss_v)) logger.info('dev acc:\t{}\tp:\t{}\tr:\t{}\tf:\t{}'.format( acc, p, r, f)) for batch in test_manager.iter_batch(shuffle=False): lengths, logits = model.run_step(sess, False, batch) _, chars, segs, tags = batch transition = model.transition.eval(session=sess) pre_seq = model.predict(logits, transition, lengths) pre_label = recover_label(pre_seq, lengths, id_to_tag) source_label = recover_label(tags, lengths, id_to_tag) predict_lists.extend(pre_label) source_tag.extend(source_label) acc_t, p_t, r_t, f_t = get_ner_fmeasure(source_tag, predict_lists, config["tag_schema"]) logger.info('test acc:\t{}\tp:\t{}\tr:\t{}\tf:\t{}'.format( acc_t, p_t, r_t, f_t)) if f > best_dev_f1: save_model(sess, model, FLAGS.ckpt_path, logger) best_dev_f1 = f best_test_f1 = f_t logger.info( 'save epoch:\t{} model with best dev f1-score'.format(i + 1)) print('****************************************************\n\n')