def train(train_data, val_data): train_dataset = MyDataset(train_data, 'train') train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size, shuffle=True) val_dataset = MyDataset(val_data, 'val') val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size, shuffle=False) model = Model().to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1) best_val_acc = 0 last_improved_epoch = 0 for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in train_loader: batch_x, batch_y = batch_x.to(device), batch_y.to(device) optimizer.zero_grad() probs = model(batch_x) train_loss = criterion(probs, batch_y) train_loss.backward() optimizer.step() cur_step += 1 if cur_step % config.train_print_step == 0: _, train_preds = torch.max(probs, 1) train_corrects = (train_preds == batch_y).sum().item() train_acc = train_corrects * 1.0 / len(batch_y) msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}' print( msg.format(cur_step, len(train_loader), train_loss.item(), train_acc)) val_loss, val_acc = evaluate(model, val_loader, criterion) if val_acc >= best_val_acc: best_val_acc = val_acc torch.save(model.state_dict(), config.model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' # msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \ # 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print( msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_acc, end_time - start_time, improved_str)) # scheduler.step() if cur_epoch - last_improved_epoch > config.patience_epoch: print("No optimization for a long time, auto-stopping...") break
def predict(): model = Model().to(device) model.load_state_dict(torch.load(config.model_save_path)) data_len = len(os.listdir(config.wav_test_path)) test_path_list = [ '{}/{}.wav'.format(config.wav_test_path, x) for x in range(0, data_len) ] test_data = pd.DataFrame({'filename': test_path_list}) test_dataset = MyDataset(test_data, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() preds_list = [] with torch.no_grad(): for batch_x, _ in test_loader: batch_x = batch_x.to(device) probs = model(batch_x) # pred = torch.argmax(output, dim=1) _, preds = torch.max(probs, 1) preds_list += [p.item() for p in preds] submission = pd.DataFrame({ "id": range(len(preds_list)), "label": preds_list }) submission.to_csv('../data/submission.csv', index=False, header=False)
def predict(): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path)) test_df = pd.read_csv(config.test_path, sep='\t') test_dataset = MyDataset(test_df, tokenizer, 'test') test_iter = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) preds_list = [] model.eval() with torch.no_grad(): for batch_x, _ in tqdm(test_iter): inputs = get_inputs(batch_x) logits = model(**inputs) preds = torch.argmax(logits, dim=1) preds_list += [config.id2label[p.item()] for p in preds] submission = pd.DataFrame({ "id": range(len(preds_list)), "label": preds_list }) submission.to_csv('submission.csv', index=False, header=False)
def model_predict(model_name): test_dataset = MyDataset(test_df, tokenizer, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size) preds_dict = dict() for fold_idx in range(config.n_splits): model = model_file.Model().to(device) model_save_path = os.path.join( config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) model.load_state_dict(torch.load(model_save_path)) pred_list = predict(model, test_loader) submission = pd.DataFrame(pred_list) submission.to_csv('{}/{}_fold{}_submission.csv'.format( config.submission_path, model_name, fold_idx), index=False, header=False) preds_dict['{}_{}'.format(model_name, fold_idx)] = pred_list pred_list = get_pred_list(preds_dict) submission = pd.DataFrame({ "id": range(len(pred_list)), "label": pred_list }) submission['label'] = submission['label'].apply( lambda x: config.id2label[x]) submission.to_csv('submission.csv', index=False, header=False)
def predict(dataset, type): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path, map_location=device)) test_dataset = MyDataset(dataset, tokenizer, type) test_iter = DataLoader(test_dataset, batch_size=config.batch_size) label_list = [] embs = dict() model.eval() with torch.no_grad(): for batch_x, batch_y in tqdm(test_iter): inputs = get_inputs(batch_x) indexs = batch_x[3] outputs = model(**inputs) pooled_outputs = outputs[1] pooled_outputs = pooled_outputs.cpu().data.numpy() for index, emb, label in zip(indexs, pooled_outputs, batch_y): index = index.item() if index in embs.keys(): embs[index] = np.vstack([embs[index], emb]) else: embs[index] = [emb] label_list.append(label.cpu().data.numpy()) new_dataset = pd.DataFrame({"emb": [embs[i] for i in range(len(label_list))], 'label': label_list}) # np.savez('../data/new_{}.npz'.format(type), emb=new_dataset['emb'], label=new_dataset['label']) np.savez('../data/bert_{}.npz'.format(type), emb=new_dataset['emb'], label=new_dataset['label'])
def predict(): model = Net(model_name).to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path)) data_len = len(os.listdir(config.image_test_path)) test_path_list = [ '{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len) ] test_data = np.array(test_path_list) test_dataset = MyDataset(test_data, test_transform, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() pred_list = [] with torch.no_grad(): for batch_x, _ in tqdm(test_loader): batch_x = batch_x.to(device) # compute output logits = model(batch_x) preds = torch.argmax(logits, dim=1) pred_list += [p.cpu().data.numpy() for p in preds] submission = pd.DataFrame({ "id": range(len(pred_list)), "label": pred_list }) submission.to_csv('submission.csv', index=False, header=False)
def predict(): model.load_state_dict(torch.load(config.model_save_path)) transform_test = transforms.Compose([ transforms.Resize([config.image_size, config.image_size]), transforms.ToTensor(), transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]) ]) train_dataset = MyDataset(config.test_path, transform_test, 'test') test_iter = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=False) model.eval() preds_list = [] with torch.no_grad(): for batch_x, _ in test_iter: batch_x = batch_x.to(device) probs = model(batch_x) # pred = torch.argmax(output, dim=1) _, preds = torch.max(probs, 1) preds_list += [p.item() for p in preds] submission = pd.DataFrame({ "id": range(len(preds_list)), "label": preds_list }) submission.to_csv('../data/densenet18_submission.csv', index=False, header=False)
def predict(): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path)) model.eval() test_df = pd.read_csv(config.test_path) # submission = pd.read_csv(config.sample_submission_path) test_dataset = MyDataset(test_df, 'test') test_iter = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) results = [] with torch.no_grad(): for batch_x, _ in tqdm(test_iter): inputs = get_inputs(batch_x) outputs = model(**inputs) logits = outputs[0] preds = torch.argmax(logits, dim=2) for pred in preds: pred = pred.cpu().data.numpy()[1:-1] # [CLS]XXXX[SEP] tags = [config.id2label[x] for x in pred] label_entities = get_entities(pred, config.id2label) pred_dict = dict() pred_dict['tag_seq'] = " ".join(tags) pred_dict['entities'] = label_entities results.append(pred_dict) test_text = [] with open("../data/test.json", 'r') as fr: for line in fr: test_text.append(json.loads(line)) submission = [] for x, y in zip(test_text, results): item = dict() item['id'] = x['id'] item['label'] = {} entities = y['entities'] words = list(x['text']) if len(entities) != 0: for subject in entities: tag = subject[0] start = subject[1] end = subject[2] word = "".join(words[start:end + 1]) if tag in item['label']: if word in item['label'][tag]: item['label'][tag][word].append([start, end]) else: item['label'][tag][word] = [[start, end]] else: item['label'][tag] = {} item['label'][tag][word] = [[start, end]] submission.append(item) with open('submission.json', 'w') as outfile: for line in submission: line = json.dumps(line, ensure_ascii=False) outfile.write(line + '\n')
def predict(): model = Net(model_name).to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path)) # data_len = len(os.listdir(config.image_test_path)) # test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)] # test_data = np.array(test_path_list) test_df = pd.read_csv(config.test_path) test_df['FileID'] = test_df['FileID'].apply(lambda x: '{}/{}.jpg'.format(config.image_test_path, x)) print('test:{}'.format(test_df.shape[0])) test_dataset = MyDataset(test_df, test_transform, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() pred_list = [] with torch.no_grad(): for batch_x, _ in tqdm(test_loader): batch_x = batch_x.to(device) # compute output probs = model(batch_x) preds = torch.argmax(probs, dim=1) pred_list += [p.item() for p in preds] submission = pd.DataFrame({"FileID": range(len(pred_list)), "SpeciesID": pred_list}) submission.to_csv('submission.csv', index=False, header=False)
def predict(): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path, map_location=device)) test_data = pd.read_csv(config.test_path) test_dataset = MyDataset(test_data, tokenizer, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() pred_list = [] with torch.no_grad(): for batch_x, _ in tqdm(test_loader): inputs = get_inputs(batch_x) logits = model(**inputs) pred_list += [p.item() for p in logits] pred_list = np.squeeze(pred_list) submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list}) submission.to_csv('submission.csv', index=False, header=False)
def predict(model): test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)] test_data = np.array(test_path_list) test_dataset = MyDataset(test_data, test_transform, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() pred_list = [] with torch.no_grad(): for batch_x, _ in tqdm(test_loader): batch_x = batch_x.to(device) # compute output probs = model(batch_x) # preds = torch.argmax(probs, dim=1) # pred_list += [p.item() for p in preds] pred_list.extend(probs.cpu().numpy()) return pred_list
def predict(): model = Model().to(config.device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path)) test_df = pd.read_csv(config.test_path) test_df.loc[:, 'selected_text'] = test_df.text.values submission = pd.read_csv(config.sample_submission_path) test_dataset = MyDataset(test_df, tokenizer, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size) pred_list = [] model.eval() with torch.no_grad(): for inputs in test_loader: tweet = inputs["tweet"] sentiment = inputs["sentiment"] ids = inputs["ids"] token_type_ids = inputs["token_type_ids"] attention_mask = inputs["attention_mask"] offsets = inputs["offsets"] ids = ids.to(config.device, dtype=torch.long) token_type_ids = token_type_ids.to(config.device, dtype=torch.long) mask = mask.to(config.device, dtype=torch.long) start_logits, end_logits = model( input_ids=ids, attention_mask=attention_mask, token_type_ids=token_type_ids, ) pred_start_idx = torch.argmax(start_logits, dim=1).cpu().data.numpy() pred_end_idx = torch.argmax(end_logits, dim=1).cpu().data.numpy() for i in range(len(tweet)): pred = get_selected_text(tweet[i], sentiment[i], pred_start_idx[i], pred_end_idx[i], offsets[i]) pred_list.append(pred) submission['selected_text'] = pred_list submission.to_csv('submission.csv', index=False)
def predict(model): #test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)] #test_path_list = ['{}/{}'.format(config.image_test_path, x) for x in os.listdir(config.image_test_path)] csv_path = '/data/beijing/dataset/BOLD/BOLD_public/val.csv' info = pd.read_csv(csv_path) test = list(info['filename']) test_path_list = [] for i in range(len(test)): if os.path.exists(config.image_test_path + '/' + test[i]): test_path_list.append(config.image_test_path + '/' + test[i]) #print(test_path_list) #exit() test_data = np.array(test_path_list) test_dataset = MyDataset(test_data, test_transform, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() pred_list = [] with torch.no_grad(): for batch_x, _ in tqdm(test_loader): batch_x = batch_x.to(device) # compute output probs = model(batch_x) # preds = torch.argmax(probs, dim=1) # pred_list += [p.item() for p in preds] pred_list.extend(probs.cpu().numpy()) #print('----------------------------------------------------------------------') #print('batch_x = ', batch_x) #print('probs = ', probs) #print(np.shape(batch_x)) #print(np.shape(probs)) #print() #exit() return pred_list,test_data
def model_predict(model_name): test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)] test_data = np.array(test_path_list) test_dataset = MyDataset(test_data, test_transform, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) preds_dict = dict() for fold_idx in range(5): model = Net(model_name).to(device) model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) model.load_state_dict(torch.load(model_save_path)) pred_list = predict(model, test_loader) submission = pd.DataFrame(pred_list) # submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list}) submission.to_csv('{}/{}_fold{}_submission.csv' .format(config.submission_path, model_name, fold_idx), index=False, header=False) preds_dict['{}_{}'.format(model_name, fold_idx)] = pred_list pred_list = get_pred_list(preds_dict) submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list}) submission.to_csv('submission.csv', index=False, header=False)
def predict(): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path, map_location=device)) test_df = pd.read_csv(config.test_path) test_dataset = MyDataset(test_df, tokenizer, 'test') test_iter = DataLoader(test_dataset, batch_size=config.batch_size) preds_list = [] model.eval() with torch.no_grad(): for batch_x, _ in tqdm(test_iter): inputs = get_inputs(batch_x) logits = model(**inputs) # preds = torch.sigmoid(logits) preds_list.extend(logits.detach().cpu().numpy()) preds_list = torch.from_numpy(np.array(preds_list)) value, arg_idx = torch.topk(preds_list, 5, dim=-1) # [N, 5] n_sample = preds_list.shape[0] ans = [] for i in range(n_sample): tmp = [] for j in range(5): if torch.sigmoid(value[i][j]) < 0.8 and j > 0: tmp.append(-1) else: tmp.append(arg_idx[i][j].item() + 1) ans.append(tmp) ans = np.array(ans) submission_df = pd.DataFrame({ 'question_id': test_df.question_id, '0': ans[:, 0], '1': ans[:, 1], '2': ans[:, 2], '3': ans[:, 3], '4': ans[:, 4], }) submission_df.to_csv('submission.csv', index=False, header=False)
def predict(): lbl2id = { "uncomfortable": 0, "diaper": 1, "awake": 2, "sleepy": 3, "hug": 4, "hungry": 5 } id2lbl = dict(zip(list(lbl2id.values()), list(lbl2id.keys()))) print(lbl2id) print(id2lbl) model = Model().to(device) model.load_state_dict(torch.load(config.model_save_path)) data_len = len(os.listdir(config.wav_test_path)) test_path_list = [ os.path.join(config.wav_test_path, f) for f in os.listdir(config.wav_test_path) ] test_data = pd.DataFrame({'filename': test_path_list}) test_dataset = MyDataset(test_data, 'test') test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False) model.eval() preds_list = [] with torch.no_grad(): for batch_x, _ in test_loader: batch_x = batch_x.to(device) probs = model(batch_x) # print(probs) pred = torch.argmax(probs, dim=1) # _, pred = torch.max(probs, 1) preds_list += [p.item() for p in pred] submission = pd.DataFrame({ "id": [f.split('/')[-1] for f in test_path_list], "label": [id2lbl[id] for id in preds_list] }) submission = submission.sort_values(by='id') submission.to_csv('submission.csv', index=False)
def predict(): model = model_file.Model().to(device) model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) model.load_state_dict(torch.load(model_save_path, map_location=device)) test_df = pd.read_csv(config.test_path) test_dataset = MyDataset(test_df, tokenizer, 'test') test_iter = DataLoader(test_dataset, batch_size=config.batch_size) preds_list = [] model.eval() with torch.no_grad(): for batch_x, _ in tqdm(test_iter): inputs = get_inputs(batch_x) logits = model(**inputs) preds = torch.argmax(logits, dim=1) preds_list += [config.id2label[p.item()] for p in preds] test_df['intent'] = preds_list test_df['slot_annotation'] = test_df['query'] test_df.to_csv('submission.csv', index=False, header=False)
def train(train_data, val_data, fold_idx=None): train_dataset = MyDataset(train_data, tokenizer) train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size) val_dataset = MyDataset(val_data, tokenizer) val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size) model = model_file.Model().to(device) criterion = nn.MSELoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-5) scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) best_val_score = 0 last_improved_epoch = 0 adjust_lr_num = 0 y_true_list = None y_pred_list = None for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in train_loader: inputs = get_inputs(batch_x, batch_y) batch_y = batch_y.to(device) optimizer.zero_grad() logits = model(**inputs) train_loss = criterion(logits.view(-1), batch_y.view(-1)) train_loss.backward() optimizer.step() cur_step += 1 y_true_list, y_pred_list = y_concatenate(y_true_list, y_pred_list, batch_y, logits) if cur_step % config.train_print_step == 0: y_pred_list = np.squeeze(y_pred_list) train_score = get_score(y_true_list, y_pred_list) msg = 'the current step: {0}/{1}, train score: {2:>6.2%}' print(msg.format(cur_step, len(train_loader), train_score)) y_true_list = None y_pred_list = None val_loss, val_score = evaluate(model, val_loader, criterion) if val_score >= best_val_score: best_val_score = val_score torch.save(model.state_dict(), model_save_path) last_improved_epoch = cur_epoch improved_str = '*' else: improved_str = '' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val score: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch: if adjust_lr_num >= model_config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") scheduler.step() last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 del model gc.collect() if fold_idx is not None: model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None): train_dataset = MyDataset(train_data) val_dataset = MyDataset(val_data) train_loader = DataLoader(train_dataset, batch_size=config.batch_size, collate_fn=collate_fn) val_loader = DataLoader(val_dataset, batch_size=config.batch_size, collate_fn=collate_fn) model = model_file.Model().to(device) optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join( config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) best_val_score = 0 last_improved_epoch = 0 adjust_lr_num = 0 y_true_list = [] y_pred_list = [] for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in tqdm(train_loader): inputs = get_inputs(batch_x, batch_y) optimizer.zero_grad() outputs = model(**inputs) train_loss, logits = outputs[:2] train_loss.backward() optimizer.step() cur_step += 1 # crf比较慢,训练crf时,注释 # y_true_list += batch_y.cpu().data.numpy().tolist() # if 'crf' in model_name: # preds, _ = model.crf._obtain_labels(logits, config.id2label, inputs['input_lens']) # else: # preds = torch.argmax(logits, dim=2) # preds = preds.cpu().data.numpy().tolist() # y_pred_list += preds # if cur_step % config.train_print_step == 0: # train_score = get_score(y_true_list, y_pred_list) # msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train score: {3:>6.2%}' # print(msg.format(cur_step, len(train_loader), train_loss.item(), train_score)) # y_true_list = [] # y_pred_list = [] # 过滤前3个step, 训练太少,可能会有问题 if cur_epoch <= 3: continue val_loss, val_score = evaluate(model, val_loader) if val_score >= best_val_score: best_val_score = val_score torch.save(model.state_dict(), model_save_path) last_improved_epoch = cur_epoch improved_str = '*' else: improved_str = '' # msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \ # 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val score: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print( msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch: if adjust_lr_num >= model_config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") scheduler.step() last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 del model gc.collect() if fold_idx is not None: model_score[fold_idx] = best_val_score
def train(): print('train:{}, val:{}'.format(len(os.listdir(config.train_path)), len(os.listdir(config.val_path)))) transform_train = transforms.Compose([ transforms.RandomResizedCrop(config.image_size), transforms.RandomHorizontalFlip(), transforms.Resize([config.image_size, config.image_size ]), # 注意 Resize 参数是 2 维,和 RandomResizedCrop 不同 transforms.ToTensor(), transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]) ]) # train_dataset = ImageFolder(config.process_train_path, transform=transform_train) train_dataset = MyDataset(config.train_path, transform_train, 'train') train_iter = DataLoader(dataset=train_dataset, batch_size=config.batch_size, shuffle=True) transform_val = transforms.Compose([ transforms.Resize([config.image_size, config.image_size]), transforms.ToTensor(), transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5]) ]) # val_dataset = ImageFolder(config.process_val_path, transform=transform_val) val_dataset = MyDataset(config.val_path, transform_val, 'val') val_iter = DataLoader(dataset=val_dataset, batch_size=config.batch_size, shuffle=False) flag = False best_val_acc = 0 cur_step = 1 last_improved_step = 0 total_step = len(train_iter) * config.epochs_num for epoch in range(config.epochs_num): for batch_x, batch_y in train_iter: # scheduler.step() model.train() batch_x, batch_y = batch_x.to(device), batch_y.to(device) optimizer.zero_grad() probs = model(batch_x) train_loss = criterion(probs, batch_y) train_loss.backward() optimizer.step() cur_step += 1 if cur_step % config.print_per_batch == 0: _, train_preds = torch.max(probs, 1) train_corrects = (train_preds == batch_y).sum().item() train_acc = train_corrects * 1.0 / len(batch_y) val_loss, val_acc = evaluate(model, val_iter) if val_acc >= best_val_acc: best_val_acc = val_acc torch.save(model.state_dict(), config.model_save_path) improved_str = '*' last_improved_step = cur_step else: improved_str = '' msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \ 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}' print( msg.format(cur_step, total_step, train_loss.item(), train_acc, val_loss, val_acc, improved_str)) if cur_step - last_improved_step > len(train_iter): print("No optimization for a long time, auto-stopping...") flag = True break if flag: break
def train(train_data, val_data, fold_idx=None): train_dataset = MyDataset(train_data, tokenizer) val_dataset = MyDataset(val_data, tokenizer) train_loader = DataLoader(train_dataset, batch_size=config.batch_size) val_loader = DataLoader(val_dataset, batch_size=config.batch_size) model = model_file.Model().to(device) # criterion = nn.CrossEntropyLoss() criterion = LabelSmoothingCrossEntropy(config.num_labels) optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) best_val_score = 0 last_improved_epoch = 0 adjust_lr_num = 0 y_true_list = [] y_pred_list = [] for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in train_loader: inputs = get_inputs(batch_x, batch_y) batch_y = batch_y.to(device) optimizer.zero_grad() outputs = model(**inputs) logits, pooled_output = outputs preds = torch.argmax(logits, dim=1) # train_loss = criterion(logits, batch_y) train_loss = criterion(logits.view(-1, config.num_labels), batch_y.view(-1)) train_loss.backward() optimizer.step() cur_step += 1 y_true_list += batch_y.cpu().data.numpy().tolist() y_pred_list += preds.cpu().data.numpy().tolist() if cur_step % config.train_print_step == 0: train_score = get_score(np.array(y_true_list), np.array(y_pred_list)) msg = 'the current step: {0}/{1}, train score: {2:>6.2%}' print(msg.format(cur_step, len(train_loader), train_score)) y_true_list = [] y_pred_list = [] val_loss, val_score = evaluate(model, val_loader, criterion) if val_score >= best_val_score: best_val_score = val_score torch.save(model.state_dict(), model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch: if adjust_lr_num >= model_config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") scheduler.step() last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 del model gc.collect() if fold_idx is not None: model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None): train_dataset = MyDataset(train_data, tokenizer) val_dataset = MyDataset(val_data, tokenizer) train_loader = DataLoader(train_dataset, batch_size=config.batch_size) val_loader = DataLoader(val_dataset, batch_size=config.batch_size) model = model_file.Model().to(device) criterion = nn.BCEWithLogitsLoss() optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join( config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) if os.path.isfile(model_save_path): print('加载之前的训练模型') try: model.load_state_dict(torch.load(model_save_path)) except: print('加载失败') best_val_loss = 100 last_improved_epoch = 0 adjust_lr_num = 0 for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in train_loader: inputs = get_inputs(batch_x, batch_y) batch_y = batch_y.to(device) optimizer.zero_grad() logits = model(**inputs) train_loss = criterion(logits, batch_y) train_loss.backward() optimizer.step() cur_step += 1 if cur_step % config.train_print_step == 0: msg = 'the current step: {0}/{1}, train loss: {2:>6.2%}' print(msg.format(cur_step, len(train_loader), train_loss)) val_loss = evaluate(model, val_loader, criterion) if val_loss <= best_val_loss: best_val_loss = val_loss torch.save(model.state_dict(), model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, cost: {3}s {4}' end_time = int(time.time()) print( msg.format(cur_epoch + 1, config.epochs_num, val_loss, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch: if adjust_lr_num >= model_config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") scheduler.step() last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 del model gc.collect()
def train(train_data, val_data, fold_idx=None): train_data = MyDataset(train_data, train_transform) train_loader = DataLoader(train_data, batch_size=config.batch_size, shuffle=True) val_data = MyDataset(val_data, val_transform) val_loader = DataLoader(val_data, batch_size=config.batch_size, shuffle=False) model = Net(model_name).to(device) # criterion = nn.CrossEntropyLoss() criterion = FocalLoss(0.5) # optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) # scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) optimizer = Ranger(model.parameters(), lr=1e-3, weight_decay=0.0005) # scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=4) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) # if os.path.isfile(model_save_path): # print('加载之前的训练模型') # model.load_state_dict(torch.load(model_save_path)) best_val_score = 0 best_val_score_cnt = 0 last_improved_epoch = 0 adjust_lr_num = 0 for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 for batch_x, batch_y in train_loader: batch_x, batch_y = batch_x.to(device), batch_y.to(device) optimizer.zero_grad() probs = model(batch_x) train_loss = criterion(probs, batch_y) train_loss.backward() optimizer.step() cur_step += 1 if cur_step % config.train_print_step == 0: train_acc = accuracy(probs, batch_y) msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}' print(msg.format(cur_step, len(train_loader), train_loss.item(), train_acc[0].item())) val_loss, val_score = evaluate(model, val_loader, criterion) if val_score >= best_val_score: if val_score == best_val_score: best_val_score_cnt += 1 best_val_score = val_score torch.save(model.state_dict(), model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch or best_val_score_cnt >= 3: if adjust_lr_num >= config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") # scheduler.step() last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 best_val_score_cnt = 0 scheduler.step() del model gc.collect() if fold_idx is not None: model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None): train_data = MyDataset(train_data, train_transform) train_loader = DataLoader(train_data, batch_size=config.batch_size, shuffle=True) val_data = MyDataset(val_data, val_transform) val_loader = DataLoader(val_data, batch_size=config.batch_size, shuffle=False) model = Net(model_name).to(device) criterion = nn.CrossEntropyLoss() # criterion = FocalLoss(0.5) optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) # optimizer = torch.optim.Adagrad(model.parameters(), lr=1e-3) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1) # config.model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) best_val_acc = 0 last_improved_epoch = 0 if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch: ', cur_epoch + 1) cur_step = 0 for batch_x, batch_y in train_loader: batch_x, batch_y = batch_x.to(device), batch_y.to(device) optimizer.zero_grad() probs = model(batch_x) train_loss = criterion(probs, batch_y) train_loss.backward() optimizer.step() cur_step += 1 if cur_step % config.train_print_step == 0: train_acc = accuracy(probs, batch_y) msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}' print(msg.format(cur_step, len(train_loader), train_loss.item(), train_acc[0].item())) val_loss, val_acc = evaluate(model, val_loader, criterion) if val_acc >= best_val_acc: best_val_acc = val_acc torch.save(model.state_dict(), model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' # msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \ # 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_acc, end_time - start_time, improved_str)) scheduler.step() if cur_epoch - last_improved_epoch > config.patience_epoch: print("No optimization for a long time, auto-stopping...") break del model gc.collect()
def train(train_data, val_data, fold_idx=None): train_dataset = MyDataset(train_data, tokenizer) val_dataset = MyDataset(val_data, tokenizer) train_loader = DataLoader(train_dataset, batch_size=config.batch_size) val_loader = DataLoader(val_dataset, batch_size=config.batch_size) model = Model().to(config.device) # optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate) # scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1) optimizer = Ranger(model.parameters(), lr=5e-5) period = int(len(train_loader) / config.train_print_step) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=period, eta_min=5e-9) # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.train_print_step, eta_min=1e-9) if fold_idx is None: print('start') model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name)) else: print('start fold: {}'.format(fold_idx + 1)) model_save_path = os.path.join( config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx)) best_val_score = 0 last_improved_epoch = 0 adjust_lr_num = 0 y_true_list = [] y_pred_list = [] for cur_epoch in range(config.epochs_num): start_time = int(time.time()) model.train() print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader))) cur_step = 0 # for batch_x, batch_y in train_loader: for inputs in train_loader: tweet = inputs["tweet"] selected_text = inputs["selected_text"] sentiment = inputs["sentiment"] ids = inputs["ids"] attention_mask = inputs["attention_mask"] token_type_ids = inputs["token_type_ids"] targets_start_idx = inputs["start_idx"] targets_end_idx = inputs["end_idx"] offsets = inputs["offsets"] # Move ids, masks, and targets to gpu while setting as torch.long ids = ids.to(config.device, dtype=torch.long) token_type_ids = token_type_ids.to(config.device, dtype=torch.long) attention_mask = attention_mask.to(config.device, dtype=torch.long) targets_start_idx = targets_start_idx.to(config.device, dtype=torch.long) targets_end_idx = targets_end_idx.to(config.device, dtype=torch.long) optimizer.zero_grad() start_logits, end_logits = model( input_ids=ids, attention_mask=attention_mask, token_type_ids=token_type_ids, ) loss = loss_fn(start_logits, end_logits, targets_start_idx, targets_end_idx) loss.backward() optimizer.step() cur_step += 1 # pred_start_idxs = torch.argmax(start_logits, dim=1).cpu().data.numpy() # pred_end_idxs = torch.argmax(end_logits, dim=1).cpu().data.numpy() # for i in range(len(tweet)): # y_true_list.append((tweet[i], selected_text[i], sentiment[i], offsets[i])) # y_pred_list.append((pred_start_idxs[i], pred_end_idxs[i])) if cur_step % config.train_print_step == 0: scheduler.step() msg = 'the current step: {0}/{1}, cost: {2}s' print( msg.format(cur_step, len(train_loader), int(time.time()) - start_time)) # train_score = get_score(y_true_list, y_pred_list) # msg = 'the current step: {0}/{1}, train score: {2:>6.2%}' # print(msg.format(cur_step, len(train_loader), train_score)) # y_true_list = [] # y_pred_list = [] val_loss, val_score = evaluate(model, val_loader) if val_score >= best_val_score: best_val_score = val_score torch.save(model.state_dict(), model_save_path) improved_str = '*' last_improved_epoch = cur_epoch else: improved_str = '' msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}' end_time = int(time.time()) print( msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score, end_time - start_time, improved_str)) if cur_epoch - last_improved_epoch >= config.patience_epoch: if adjust_lr_num >= model_config.adjust_lr_num: print("No optimization for a long time, auto stopping...") break print("No optimization for a long time, adjust lr...") last_improved_epoch = cur_epoch # 加上,不然会连续更新的 adjust_lr_num += 1 del model gc.collect() if fold_idx is not None: model_score[fold_idx] = best_val_score