def test(): for epoch in range(args.epochs): # 载入模型 output_path = args.output_path + str(epoch) args.model_path = os.path.join(output_path, "pytorch_model.bin") tokenizer, model = load_model(args) # 载入测试集数据 data_path = args.data_path + str(epoch) test_data_path = os.path.join(data_path, 'test.csv') test_loader = load_data(tokenizer, args, test_data_path, "test") # 3263 logger.info("Testing data has been loaded!") # 得到测试结果 running = Running(model, args) outputs = running.test(test_loader) # 写入数据 outputs_df = pd.read_csv( os.path.join(args.raw_data_path, "sample_submission.csv")) outputs_df['target_0'] = outputs[:, 0] outputs_df['target_1'] = outputs[:, 1] outputs_df['target_2'] = outputs[:, 2] outputs_df[['id', 'target_0', 'target_1', 'target_2']].to_csv(os.path.join(output_path, "sub.csv"), index=False) logger.info('sub ' + str(epoch) + ' has been written.')
def train(): # 训练epochs轮 for epoch in range(args.epochs): # 载入模型 tokenizer, model = load_model(args) # 读取数据 data_path = args.data_path + str(epoch) train_data_path = os.path.join(data_path, 'train.csv') train_loader = load_data(tokenizer, args, train_data_path, "train") evaluate_data_path = os.path.join(data_path, 'dev.csv') evaluate_loader = load_data(tokenizer, args, evaluate_data_path, "evaluate") logger.info("Training data has been loaded!") # 训练 running = Running(model, args) model = running.model # 定义优化器 base_opt = torch.optim.SGD(model.parameters(), lr=0.1) optimizer = torchcontrib.optim.SWA(base_opt, swa_start=10, swa_freq=5, swa_lr=0.05) # optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup, num_training_steps=(args.epochs * len( train_loader) / args.batch_accumulation)) # 训练 running.train(train_loader, evaluate_loader, optimizer, scheduler, epoch) torch.cuda.empty_cache() # 释放显存 gc.collect() # 垃圾回收 # 模型的权重设置为它们的SWA平均值 optimizer.swap_swa_sgd()
def train(): # 训练epochs轮 for epoch in range(args.epochs): # 载入模型 tokenizer, model = load_model(args) # 读取数据 data_path = args.data_path + str(epoch) train_data_path = os.path.join(data_path, 'train.csv') train_loader = load_data(tokenizer, args, train_data_path, "train") evaluate_data_path = os.path.join(data_path, 'dev.csv') evaluate_loader = load_data(tokenizer, args, evaluate_data_path, "evaluate") logger.info("Training data has been loaded!") # 训练 running = Running(model, args) running.train(train_loader, evaluate_loader, epoch) # 释放显存 torch.cuda.empty_cache() # 垃圾回收 gc.collect()