def inference_random(): # 加载验证集验证 model = ClassificationModel(len(cfg.char2idx)) model = load_custom_model(model, cfg.save_model_path).to(cfg.device) tokenizer = Tokenizer(cfg.char2idx) error = 0 with open(cfg.test_data_path, 'r', encoding='utf-8') as f: lines = f.readlines() for line in lines: pairs = line.split('\t') label, text = pairs[0], pairs[1] input_index, _ = tokenizer.encode(text, max_length=cfg.max_seq_len) inputs = torch.tensor(input_index).unsqueeze(0) inputs_mask = (inputs > 0).to(torch.float32) with torch.no_grad(): scores = model(inputs, inputs_mask) prediction = scores.argmax(-1).item() if prediction != int(label): print(scores[:, int(label)].item()) print(label) print(text) print('-' * 50) error += 1 print(error)
def train(): # 加载数据 char2idx, keep_tokens = load_chinese_base_vocab(cfg.vocab_path) tokenizer = Tokenizer(char2idx) # train_data = glob(cfg.train_data_path + '*')[16 * 1000 * 35:16 * 1000 * 40] train_data = glob(cfg.train_data_path + '*')[8 * 5000 * 5:8 * 5000 * 10] train_dataset = CustomDataset(train_data, tokenizer, cfg.max_seq_len) train_dataloader = DataLoader(train_dataset, batch_size=cfg.batch_size, collate_fn=padding, shuffle=True, num_workers=4, pin_memory=True) # # debug # train_data = glob(cfg.test_data_path + '*')[:8 * 5000 * 5] # train_dataset = CustomDataset(train_data, tokenizer, cfg.max_seq_len) # train_dataloader = DataLoader(train_dataset, batch_size=cfg.batch_size, collate_fn=padding) # # debug # 加载模型 model = CustomUnilmModel(len(char2idx)) # model = load_pretrained_bert(model, cfg.pretrained_model_path, keep_tokens=keep_tokens).to(cfg.device) model = load_custom_model(model, cfg.save_model_path).to(cfg.device) loss_function = nn.CrossEntropyLoss(ignore_index=0).to(cfg.device) optimizer = torch.optim.Adam(model.parameters(), lr=cfg.learn_rate) # 迭代训练 iteration, train_loss = 0, 0 model.train() for inputs, token_type, targets in tqdm(train_dataloader, position=0, leave=True): attention_mask = unilm_mask(inputs, token_type).to(cfg.device) inputs, token_type, targets = inputs.to(cfg.device), token_type.to( cfg.device), targets.to(cfg.device) prediction = model(inputs, token_type, attention_mask) loss = loss_function( prediction[:, :-1, :].reshape(-1, prediction.shape[-1]), targets.reshape(-1)) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() iteration += 1 if iteration % cfg.print_loss_steps == 0: eval_loss = evaluate(model, tokenizer, loss_function) print('') print('train_loss:{}'.format(train_loss / cfg.print_loss_steps)) print('evalu_loss:{}'.format(eval_loss)) test_string(s1, tokenizer, model) test_string(s2, tokenizer, model) model.train() train_loss = 0 if iteration % cfg.save_model_steps == 0: torch.save(model.state_dict(), cfg.save_model_path)
def train(): # 加载数据 tokenizer = Tokenizer(cfg.char2idx) train_dataset = CustomDataset(cfg.train_data_path, tokenizer, cfg) train_dataloader = DataLoader(train_dataset, batch_size=cfg.batch_size, collate_fn=padding, shuffle=True, num_workers=4, pin_memory=True) model = ClassificationModel(len(cfg.char2idx)) # model = load_pretrained_bert(model, cfg.pretrained_model_path, keep_tokens=cfg.keep_tokens).to(cfg.device) model = load_custom_model(model, cfg.save_model_path).to(cfg.device) loss_function = nn.CrossEntropyLoss().to(cfg.device) optimizer = torch.optim.Adam(model.parameters(), lr=cfg.learn_rate) # 迭代训练 iteration, train_loss = 0, 0 model.train() for inputs, mask, targets in tqdm(train_dataloader, position=0, leave=True): inputs, mask, targets = inputs.to(cfg.device), mask.to( cfg.device), targets.to(cfg.device) prediction = model(inputs, mask) loss = loss_function(prediction, targets.reshape(-1)) optimizer.zero_grad() loss.backward() optimizer.step() train_loss += loss.item() iteration += 1 if iteration % cfg.print_loss_steps == 0: eval_loss = evaluate(model, tokenizer, loss_function) print('') print('train_loss:{}'.format(train_loss / cfg.print_loss_steps)) print('evalu_loss:{}'.format(eval_loss)) accuracy(model, tokenizer, cfg.valid_data_path) accuracy(model, tokenizer, cfg.test_data_path) model.train() train_loss = 0 if iteration % cfg.save_model_steps == 0: torch.save(model.state_dict(), cfg.save_model_path)
device_count = {'CPU' : 1, 'GPU' : 0} ) session = Session(config=device_config) K.set_session(session) else: os.environ["CUDA_VISIBLE_DEVICES"] = str(G) from keras.models import Model from keras.models import model_from_json, load_model from utils import init_predictor, DecodeCTCPred, Readf, edit_distance, normalized_edit_distance, \ BilinearInterpolation, get_lexicon, load_custom_model, open_img, norm, parse_mjsynth prng = RandomState(random_state) model = load_custom_model(model_path, model_name='/model.json', weights="/final_weights.h5") model = init_predictor(model) classes = {j: i for i, j in enumerate(get_lexicon())} inverse_classes = {v: k for k, v in classes.items()} decoder = DecodeCTCPred(top_paths=1, beam_width=10, inverse_classes=inverse_classes) img_size = (imgh, imgW) + (1, ) if validate: if mjsynth: fnames = open(os.path.join(image_path, val_fname), "r").readlines() fnames = np.array(parse_mjsynth(image_path, fnames))