def train(train_data, val_data):
train_dataset = MyDataset(train_data, 'train')
train_loader = DataLoader(dataset=train_dataset,
batch_size=config.batch_size,
shuffle=True)
val_dataset = MyDataset(val_data, 'val')
val_loader = DataLoader(dataset=val_dataset,
batch_size=config.batch_size,
shuffle=False)
model = Model().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)
best_val_acc = 0
last_improved_epoch = 0
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in train_loader:
batch_x, batch_y = batch_x.to(device), batch_y.to(device)
optimizer.zero_grad()
probs = model(batch_x)
train_loss = criterion(probs, batch_y)
train_loss.backward()
optimizer.step()
cur_step += 1
if cur_step % config.train_print_step == 0:
_, train_preds = torch.max(probs, 1)
train_corrects = (train_preds == batch_y).sum().item()
train_acc = train_corrects * 1.0 / len(batch_y)
msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}'
print(
msg.format(cur_step, len(train_loader), train_loss.item(),
train_acc))
val_loss, val_acc = evaluate(model, val_loader, criterion)
if val_acc >= best_val_acc:
best_val_acc = val_acc
torch.save(model.state_dict(), config.model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
# msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \
# 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}'
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(
msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_acc,
end_time - start_time, improved_str))
# scheduler.step()
if cur_epoch - last_improved_epoch > config.patience_epoch:
print("No optimization for a long time, auto-stopping...")
break
def predict():
model = Model().to(device)
model.load_state_dict(torch.load(config.model_save_path))
data_len = len(os.listdir(config.wav_test_path))
test_path_list = [
'{}/{}.wav'.format(config.wav_test_path, x)
for x in range(0, data_len)
]
test_data = pd.DataFrame({'filename': test_path_list})
test_dataset = MyDataset(test_data, 'test')
test_loader = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False)
model.eval()
preds_list = []
with torch.no_grad():
for batch_x, _ in test_loader:
batch_x = batch_x.to(device)
probs = model(batch_x)
# pred = torch.argmax(output, dim=1)
_, preds = torch.max(probs, 1)
preds_list += [p.item() for p in preds]
submission = pd.DataFrame({
"id": range(len(preds_list)),
"label": preds_list
})
submission.to_csv('../data/submission.csv', index=False, header=False)
def predict():
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path))
test_df = pd.read_csv(config.test_path, sep='\t')
test_dataset = MyDataset(test_df, tokenizer, 'test')
test_iter = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False)
preds_list = []
model.eval()
with torch.no_grad():
for batch_x, _ in tqdm(test_iter):
inputs = get_inputs(batch_x)
logits = model(**inputs)
preds = torch.argmax(logits, dim=1)
preds_list += [config.id2label[p.item()] for p in preds]
submission = pd.DataFrame({
"id": range(len(preds_list)),
"label": preds_list
})
submission.to_csv('submission.csv', index=False, header=False)
def model_predict(model_name):
test_dataset = MyDataset(test_df, tokenizer, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size)
preds_dict = dict()
for fold_idx in range(config.n_splits):
model = model_file.Model().to(device)
model_save_path = os.path.join(
config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
model.load_state_dict(torch.load(model_save_path))
pred_list = predict(model, test_loader)
submission = pd.DataFrame(pred_list)
submission.to_csv('{}/{}_fold{}_submission.csv'.format(
config.submission_path, model_name, fold_idx),
index=False,
header=False)
preds_dict['{}_{}'.format(model_name, fold_idx)] = pred_list
pred_list = get_pred_list(preds_dict)
submission = pd.DataFrame({
"id": range(len(pred_list)),
"label": pred_list
})
submission['label'] = submission['label'].apply(
lambda x: config.id2label[x])
submission.to_csv('submission.csv', index=False, header=False)
def predict(dataset, type):
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path, map_location=device))
test_dataset = MyDataset(dataset, tokenizer, type)
test_iter = DataLoader(test_dataset, batch_size=config.batch_size)
label_list = []
embs = dict()
model.eval()
with torch.no_grad():
for batch_x, batch_y in tqdm(test_iter):
inputs = get_inputs(batch_x)
indexs = batch_x[3]
outputs = model(**inputs)
pooled_outputs = outputs[1]
pooled_outputs = pooled_outputs.cpu().data.numpy()
for index, emb, label in zip(indexs, pooled_outputs, batch_y):
index = index.item()
if index in embs.keys():
embs[index] = np.vstack([embs[index], emb])
else:
embs[index] = [emb]
label_list.append(label.cpu().data.numpy())
new_dataset = pd.DataFrame({"emb": [embs[i] for i in range(len(label_list))], 'label': label_list})
# np.savez('../data/new_{}.npz'.format(type), emb=new_dataset['emb'], label=new_dataset['label'])
np.savez('../data/bert_{}.npz'.format(type), emb=new_dataset['emb'], label=new_dataset['label'])
def predict():
model = Net(model_name).to(device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path))
data_len = len(os.listdir(config.image_test_path))
test_path_list = [
'{}/{}.jpg'.format(config.image_test_path, x)
for x in range(0, data_len)
]
test_data = np.array(test_path_list)
test_dataset = MyDataset(test_data, test_transform, 'test')
test_loader = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False)
model.eval()
pred_list = []
with torch.no_grad():
for batch_x, _ in tqdm(test_loader):
batch_x = batch_x.to(device)
# compute output
logits = model(batch_x)
preds = torch.argmax(logits, dim=1)
pred_list += [p.cpu().data.numpy() for p in preds]
submission = pd.DataFrame({
"id": range(len(pred_list)),
"label": pred_list
})
submission.to_csv('submission.csv', index=False, header=False)
def predict():
model.load_state_dict(torch.load(config.model_save_path))
transform_test = transforms.Compose([
transforms.Resize([config.image_size, config.image_size]),
transforms.ToTensor(),
transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])
])
train_dataset = MyDataset(config.test_path, transform_test, 'test')
test_iter = DataLoader(train_dataset,
batch_size=config.batch_size,
shuffle=False)
model.eval()
preds_list = []
with torch.no_grad():
for batch_x, _ in test_iter:
batch_x = batch_x.to(device)
probs = model(batch_x)
# pred = torch.argmax(output, dim=1)
_, preds = torch.max(probs, 1)
preds_list += [p.item() for p in preds]
submission = pd.DataFrame({
"id": range(len(preds_list)),
"label": preds_list
})
submission.to_csv('../data/densenet18_submission.csv',
index=False,
header=False)
def predict():
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path))
model.eval()
test_df = pd.read_csv(config.test_path)
# submission = pd.read_csv(config.sample_submission_path)
test_dataset = MyDataset(test_df, 'test')
test_iter = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False)
results = []
with torch.no_grad():
for batch_x, _ in tqdm(test_iter):
inputs = get_inputs(batch_x)
outputs = model(**inputs)
logits = outputs[0]
preds = torch.argmax(logits, dim=2)
for pred in preds:
pred = pred.cpu().data.numpy()[1:-1] # [CLS]XXXX[SEP]
tags = [config.id2label[x] for x in pred]
label_entities = get_entities(pred, config.id2label)
pred_dict = dict()
pred_dict['tag_seq'] = " ".join(tags)
pred_dict['entities'] = label_entities
results.append(pred_dict)
test_text = []
with open("../data/test.json", 'r') as fr:
for line in fr:
test_text.append(json.loads(line))
submission = []
for x, y in zip(test_text, results):
item = dict()
item['id'] = x['id']
item['label'] = {}
entities = y['entities']
words = list(x['text'])
if len(entities) != 0:
for subject in entities:
tag = subject[0]
start = subject[1]
end = subject[2]
word = "".join(words[start:end + 1])
if tag in item['label']:
if word in item['label'][tag]:
item['label'][tag][word].append([start, end])
else:
item['label'][tag][word] = [[start, end]]
else:
item['label'][tag] = {}
item['label'][tag][word] = [[start, end]]
submission.append(item)
with open('submission.json', 'w') as outfile:
for line in submission:
line = json.dumps(line, ensure_ascii=False)
outfile.write(line + '\n')
def predict():
model = Net(model_name).to(device)
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path))
# data_len = len(os.listdir(config.image_test_path))
# test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)]
# test_data = np.array(test_path_list)
test_df = pd.read_csv(config.test_path)
test_df['FileID'] = test_df['FileID'].apply(lambda x: '{}/{}.jpg'.format(config.image_test_path, x))
print('test:{}'.format(test_df.shape[0]))
test_dataset = MyDataset(test_df, test_transform, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)
model.eval()
pred_list = []
with torch.no_grad():
for batch_x, _ in tqdm(test_loader):
batch_x = batch_x.to(device)
# compute output
probs = model(batch_x)
preds = torch.argmax(probs, dim=1)
pred_list += [p.item() for p in preds]
submission = pd.DataFrame({"FileID": range(len(pred_list)), "SpeciesID": pred_list})
submission.to_csv('submission.csv', index=False, header=False)
def predict():
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path, map_location=device))
test_data = pd.read_csv(config.test_path)
test_dataset = MyDataset(test_data, tokenizer, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)
model.eval()
pred_list = []
with torch.no_grad():
for batch_x, _ in tqdm(test_loader):
inputs = get_inputs(batch_x)
logits = model(**inputs)
pred_list += [p.item() for p in logits]
pred_list = np.squeeze(pred_list)
submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list})
submission.to_csv('submission.csv', index=False, header=False)
def predict(model):
test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)]
test_data = np.array(test_path_list)
test_dataset = MyDataset(test_data, test_transform, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)
model.eval()
pred_list = []
with torch.no_grad():
for batch_x, _ in tqdm(test_loader):
batch_x = batch_x.to(device)
# compute output
probs = model(batch_x)
# preds = torch.argmax(probs, dim=1)
# pred_list += [p.item() for p in preds]
pred_list.extend(probs.cpu().numpy())
return pred_list
def predict():
model = Model().to(config.device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path))
test_df = pd.read_csv(config.test_path)
test_df.loc[:, 'selected_text'] = test_df.text.values
submission = pd.read_csv(config.sample_submission_path)
test_dataset = MyDataset(test_df, tokenizer, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size)
pred_list = []
model.eval()
with torch.no_grad():
for inputs in test_loader:
tweet = inputs["tweet"]
sentiment = inputs["sentiment"]
ids = inputs["ids"]
token_type_ids = inputs["token_type_ids"]
attention_mask = inputs["attention_mask"]
offsets = inputs["offsets"]
ids = ids.to(config.device, dtype=torch.long)
token_type_ids = token_type_ids.to(config.device, dtype=torch.long)
mask = mask.to(config.device, dtype=torch.long)
start_logits, end_logits = model(
input_ids=ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
)
pred_start_idx = torch.argmax(start_logits,
dim=1).cpu().data.numpy()
pred_end_idx = torch.argmax(end_logits, dim=1).cpu().data.numpy()
for i in range(len(tweet)):
pred = get_selected_text(tweet[i], sentiment[i],
pred_start_idx[i], pred_end_idx[i],
offsets[i])
pred_list.append(pred)
submission['selected_text'] = pred_list
submission.to_csv('submission.csv', index=False)
def predict(model):
#test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)]
#test_path_list = ['{}/{}'.format(config.image_test_path, x) for x in os.listdir(config.image_test_path)]
csv_path = '/data/beijing/dataset/BOLD/BOLD_public/val.csv'
info = pd.read_csv(csv_path)
test = list(info['filename'])
test_path_list = []
for i in range(len(test)):
if os.path.exists(config.image_test_path + '/' + test[i]):
test_path_list.append(config.image_test_path + '/' + test[i])
#print(test_path_list)
#exit()
test_data = np.array(test_path_list)
test_dataset = MyDataset(test_data, test_transform, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)
model.eval()
pred_list = []
with torch.no_grad():
for batch_x, _ in tqdm(test_loader):
batch_x = batch_x.to(device)
# compute output
probs = model(batch_x)
# preds = torch.argmax(probs, dim=1)
# pred_list += [p.item() for p in preds]
pred_list.extend(probs.cpu().numpy())
#print('----------------------------------------------------------------------')
#print('batch_x = ', batch_x)
#print('probs = ', probs)
#print(np.shape(batch_x))
#print(np.shape(probs))
#print()
#exit()
return pred_list,test_data
def model_predict(model_name):
test_path_list = ['{}/{}.jpg'.format(config.image_test_path, x) for x in range(0, data_len)]
test_data = np.array(test_path_list)
test_dataset = MyDataset(test_data, test_transform, 'test')
test_loader = DataLoader(test_dataset, batch_size=config.batch_size, shuffle=False)
preds_dict = dict()
for fold_idx in range(5):
model = Net(model_name).to(device)
model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
model.load_state_dict(torch.load(model_save_path))
pred_list = predict(model, test_loader)
submission = pd.DataFrame(pred_list)
# submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list})
submission.to_csv('{}/{}_fold{}_submission.csv'
.format(config.submission_path, model_name, fold_idx), index=False, header=False)
preds_dict['{}_{}'.format(model_name, fold_idx)] = pred_list
pred_list = get_pred_list(preds_dict)
submission = pd.DataFrame({"id": range(len(pred_list)), "label": pred_list})
submission.to_csv('submission.csv', index=False, header=False)
def predict():
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path, map_location=device))
test_df = pd.read_csv(config.test_path)
test_dataset = MyDataset(test_df, tokenizer, 'test')
test_iter = DataLoader(test_dataset, batch_size=config.batch_size)
preds_list = []
model.eval()
with torch.no_grad():
for batch_x, _ in tqdm(test_iter):
inputs = get_inputs(batch_x)
logits = model(**inputs)
# preds = torch.sigmoid(logits)
preds_list.extend(logits.detach().cpu().numpy())
preds_list = torch.from_numpy(np.array(preds_list))
value, arg_idx = torch.topk(preds_list, 5, dim=-1) # [N, 5]
n_sample = preds_list.shape[0]
ans = []
for i in range(n_sample):
tmp = []
for j in range(5):
if torch.sigmoid(value[i][j]) < 0.8 and j > 0:
tmp.append(-1)
else:
tmp.append(arg_idx[i][j].item() + 1)
ans.append(tmp)
ans = np.array(ans)
submission_df = pd.DataFrame({
'question_id': test_df.question_id,
'0': ans[:, 0],
'1': ans[:, 1],
'2': ans[:, 2],
'3': ans[:, 3],
'4': ans[:, 4],
})
submission_df.to_csv('submission.csv', index=False, header=False)
def predict():
lbl2id = {
"uncomfortable": 0,
"diaper": 1,
"awake": 2,
"sleepy": 3,
"hug": 4,
"hungry": 5
}
id2lbl = dict(zip(list(lbl2id.values()), list(lbl2id.keys())))
print(lbl2id)
print(id2lbl)
model = Model().to(device)
model.load_state_dict(torch.load(config.model_save_path))
data_len = len(os.listdir(config.wav_test_path))
test_path_list = [
os.path.join(config.wav_test_path, f)
for f in os.listdir(config.wav_test_path)
]
test_data = pd.DataFrame({'filename': test_path_list})
test_dataset = MyDataset(test_data, 'test')
test_loader = DataLoader(test_dataset,
batch_size=config.batch_size,
shuffle=False)
model.eval()
preds_list = []
with torch.no_grad():
for batch_x, _ in test_loader:
batch_x = batch_x.to(device)
probs = model(batch_x)
# print(probs)
pred = torch.argmax(probs, dim=1)
# _, pred = torch.max(probs, 1)
preds_list += [p.item() for p in pred]
submission = pd.DataFrame({
"id": [f.split('/')[-1] for f in test_path_list],
"label": [id2lbl[id] for id in preds_list]
})
submission = submission.sort_values(by='id')
submission.to_csv('submission.csv', index=False)
def predict():
model = model_file.Model().to(device)
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
model.load_state_dict(torch.load(model_save_path, map_location=device))
test_df = pd.read_csv(config.test_path)
test_dataset = MyDataset(test_df, tokenizer, 'test')
test_iter = DataLoader(test_dataset, batch_size=config.batch_size)
preds_list = []
model.eval()
with torch.no_grad():
for batch_x, _ in tqdm(test_iter):
inputs = get_inputs(batch_x)
logits = model(**inputs)
preds = torch.argmax(logits, dim=1)
preds_list += [config.id2label[p.item()] for p in preds]
test_df['intent'] = preds_list
test_df['slot_annotation'] = test_df['query']
test_df.to_csv('submission.csv', index=False, header=False)
def train(train_data, val_data, fold_idx=None):
train_dataset = MyDataset(train_data, tokenizer)
train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size)
val_dataset = MyDataset(val_data, tokenizer)
val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size)
model = model_file.Model().to(device)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
best_val_score = 0
last_improved_epoch = 0
adjust_lr_num = 0
y_true_list = None
y_pred_list = None
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in train_loader:
inputs = get_inputs(batch_x, batch_y)
batch_y = batch_y.to(device)
optimizer.zero_grad()
logits = model(**inputs)
train_loss = criterion(logits.view(-1), batch_y.view(-1))
train_loss.backward()
optimizer.step()
cur_step += 1
y_true_list, y_pred_list = y_concatenate(y_true_list, y_pred_list, batch_y, logits)
if cur_step % config.train_print_step == 0:
y_pred_list = np.squeeze(y_pred_list)
train_score = get_score(y_true_list, y_pred_list)
msg = 'the current step: {0}/{1}, train score: {2:>6.2%}'
print(msg.format(cur_step, len(train_loader), train_score))
y_true_list = None
y_pred_list = None
val_loss, val_score = evaluate(model, val_loader, criterion)
if val_score >= best_val_score:
best_val_score = val_score
torch.save(model.state_dict(), model_save_path)
last_improved_epoch = cur_epoch
improved_str = '*'
else:
improved_str = ''
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val score: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch:
if adjust_lr_num >= model_config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
scheduler.step()
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
del model
gc.collect()
if fold_idx is not None:
model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None):
train_dataset = MyDataset(train_data)
val_dataset = MyDataset(val_data)
train_loader = DataLoader(train_dataset,
batch_size=config.batch_size,
collate_fn=collate_fn)
val_loader = DataLoader(val_dataset,
batch_size=config.batch_size,
collate_fn=collate_fn)
model = model_file.Model().to(device)
optimizer = torch.optim.Adam(model.parameters(),
lr=model_config.learning_rate)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(
config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
best_val_score = 0
last_improved_epoch = 0
adjust_lr_num = 0
y_true_list = []
y_pred_list = []
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in tqdm(train_loader):
inputs = get_inputs(batch_x, batch_y)
optimizer.zero_grad()
outputs = model(**inputs)
train_loss, logits = outputs[:2]
train_loss.backward()
optimizer.step()
cur_step += 1
# crf比较慢，训练crf时，注释
# y_true_list += batch_y.cpu().data.numpy().tolist()
# if 'crf' in model_name:
# preds, _ = model.crf._obtain_labels(logits, config.id2label, inputs['input_lens'])
# else:
# preds = torch.argmax(logits, dim=2)
# preds = preds.cpu().data.numpy().tolist()
# y_pred_list += preds
# if cur_step % config.train_print_step == 0:
# train_score = get_score(y_true_list, y_pred_list)
# msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train score: {3:>6.2%}'
# print(msg.format(cur_step, len(train_loader), train_loss.item(), train_score))
# y_true_list = []
# y_pred_list = []
# 过滤前3个step, 训练太少，可能会有问题
if cur_epoch <= 3:
continue
val_loss, val_score = evaluate(model, val_loader)
if val_score >= best_val_score:
best_val_score = val_score
torch.save(model.state_dict(), model_save_path)
last_improved_epoch = cur_epoch
improved_str = '*'
else:
improved_str = ''
# msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \
# 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}'
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val score: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(
msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch:
if adjust_lr_num >= model_config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
scheduler.step()
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
del model
gc.collect()
if fold_idx is not None:
model_score[fold_idx] = best_val_score
def train():
print('train:{}, val:{}'.format(len(os.listdir(config.train_path)),
len(os.listdir(config.val_path))))
transform_train = transforms.Compose([
transforms.RandomResizedCrop(config.image_size),
transforms.RandomHorizontalFlip(),
transforms.Resize([config.image_size, config.image_size
]), # 注意 Resize 参数是 2 维，和 RandomResizedCrop 不同
transforms.ToTensor(),
transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])
])
# train_dataset = ImageFolder(config.process_train_path, transform=transform_train)
train_dataset = MyDataset(config.train_path, transform_train, 'train')
train_iter = DataLoader(dataset=train_dataset,
batch_size=config.batch_size,
shuffle=True)
transform_val = transforms.Compose([
transforms.Resize([config.image_size, config.image_size]),
transforms.ToTensor(),
transforms.Normalize(mean=[.5, .5, .5], std=[.5, .5, .5])
])
# val_dataset = ImageFolder(config.process_val_path, transform=transform_val)
val_dataset = MyDataset(config.val_path, transform_val, 'val')
val_iter = DataLoader(dataset=val_dataset,
batch_size=config.batch_size,
shuffle=False)
flag = False
best_val_acc = 0
cur_step = 1
last_improved_step = 0
total_step = len(train_iter) * config.epochs_num
for epoch in range(config.epochs_num):
for batch_x, batch_y in train_iter:
# scheduler.step()
model.train()
batch_x, batch_y = batch_x.to(device), batch_y.to(device)
optimizer.zero_grad()
probs = model(batch_x)
train_loss = criterion(probs, batch_y)
train_loss.backward()
optimizer.step()
cur_step += 1
if cur_step % config.print_per_batch == 0:
_, train_preds = torch.max(probs, 1)
train_corrects = (train_preds == batch_y).sum().item()
train_acc = train_corrects * 1.0 / len(batch_y)
val_loss, val_acc = evaluate(model, val_iter)
if val_acc >= best_val_acc:
best_val_acc = val_acc
torch.save(model.state_dict(), config.model_save_path)
improved_str = '*'
last_improved_step = cur_step
else:
improved_str = ''
msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \
'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}'
print(
msg.format(cur_step, total_step, train_loss.item(),
train_acc, val_loss, val_acc, improved_str))
if cur_step - last_improved_step > len(train_iter):
print("No optimization for a long time, auto-stopping...")
flag = True
break
if flag:
break
def train(train_data, val_data, fold_idx=None):
train_dataset = MyDataset(train_data, tokenizer)
val_dataset = MyDataset(val_data, tokenizer)
train_loader = DataLoader(train_dataset, batch_size=config.batch_size)
val_loader = DataLoader(val_dataset, batch_size=config.batch_size)
model = model_file.Model().to(device)
# criterion = nn.CrossEntropyLoss()
criterion = LabelSmoothingCrossEntropy(config.num_labels)
optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
best_val_score = 0
last_improved_epoch = 0
adjust_lr_num = 0
y_true_list = []
y_pred_list = []
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in train_loader:
inputs = get_inputs(batch_x, batch_y)
batch_y = batch_y.to(device)
optimizer.zero_grad()
outputs = model(**inputs)
logits, pooled_output = outputs
preds = torch.argmax(logits, dim=1)
# train_loss = criterion(logits, batch_y)
train_loss = criterion(logits.view(-1, config.num_labels), batch_y.view(-1))
train_loss.backward()
optimizer.step()
cur_step += 1
y_true_list += batch_y.cpu().data.numpy().tolist()
y_pred_list += preds.cpu().data.numpy().tolist()
if cur_step % config.train_print_step == 0:
train_score = get_score(np.array(y_true_list), np.array(y_pred_list))
msg = 'the current step: {0}/{1}, train score: {2:>6.2%}'
print(msg.format(cur_step, len(train_loader), train_score))
y_true_list = []
y_pred_list = []
val_loss, val_score = evaluate(model, val_loader, criterion)
if val_score >= best_val_score:
best_val_score = val_score
torch.save(model.state_dict(), model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch:
if adjust_lr_num >= model_config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
scheduler.step()
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
del model
gc.collect()
if fold_idx is not None:
model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None):
train_dataset = MyDataset(train_data, tokenizer)
val_dataset = MyDataset(val_data, tokenizer)
train_loader = DataLoader(train_dataset, batch_size=config.batch_size)
val_loader = DataLoader(val_dataset, batch_size=config.batch_size)
model = model_file.Model().to(device)
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=model_config.learning_rate)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(
config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
if os.path.isfile(model_save_path):
print('加载之前的训练模型')
try:
model.load_state_dict(torch.load(model_save_path))
except:
print('加载失败')
best_val_loss = 100
last_improved_epoch = 0
adjust_lr_num = 0
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in train_loader:
inputs = get_inputs(batch_x, batch_y)
batch_y = batch_y.to(device)
optimizer.zero_grad()
logits = model(**inputs)
train_loss = criterion(logits, batch_y)
train_loss.backward()
optimizer.step()
cur_step += 1
if cur_step % config.train_print_step == 0:
msg = 'the current step: {0}/{1}, train loss: {2:>6.2%}'
print(msg.format(cur_step, len(train_loader), train_loss))
val_loss = evaluate(model, val_loader, criterion)
if val_loss <= best_val_loss:
best_val_loss = val_loss
torch.save(model.state_dict(), model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, cost: {3}s {4}'
end_time = int(time.time())
print(
msg.format(cur_epoch + 1, config.epochs_num, val_loss,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch:
if adjust_lr_num >= model_config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
scheduler.step()
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
del model
gc.collect()
def train(train_data, val_data, fold_idx=None):
train_data = MyDataset(train_data, train_transform)
train_loader = DataLoader(train_data, batch_size=config.batch_size, shuffle=True)
val_data = MyDataset(val_data, val_transform)
val_loader = DataLoader(val_data, batch_size=config.batch_size, shuffle=False)
model = Net(model_name).to(device)
# criterion = nn.CrossEntropyLoss()
criterion = FocalLoss(0.5)
# optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
# scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
optimizer = Ranger(model.parameters(), lr=1e-3, weight_decay=0.0005)
# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=4)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
# if os.path.isfile(model_save_path):
# print('加载之前的训练模型')
# model.load_state_dict(torch.load(model_save_path))
best_val_score = 0
best_val_score_cnt = 0
last_improved_epoch = 0
adjust_lr_num = 0
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
for batch_x, batch_y in train_loader:
batch_x, batch_y = batch_x.to(device), batch_y.to(device)
optimizer.zero_grad()
probs = model(batch_x)
train_loss = criterion(probs, batch_y)
train_loss.backward()
optimizer.step()
cur_step += 1
if cur_step % config.train_print_step == 0:
train_acc = accuracy(probs, batch_y)
msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}'
print(msg.format(cur_step, len(train_loader), train_loss.item(), train_acc[0].item()))
val_loss, val_score = evaluate(model, val_loader, criterion)
if val_score >= best_val_score:
if val_score == best_val_score:
best_val_score_cnt += 1
best_val_score = val_score
torch.save(model.state_dict(), model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch or best_val_score_cnt >= 3:
if adjust_lr_num >= config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
# scheduler.step()
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
best_val_score_cnt = 0
scheduler.step()
del model
gc.collect()
if fold_idx is not None:
model_score[fold_idx] = best_val_score
def train(train_data, val_data, fold_idx=None):
train_data = MyDataset(train_data, train_transform)
train_loader = DataLoader(train_data, batch_size=config.batch_size, shuffle=True)
val_data = MyDataset(val_data, val_transform)
val_loader = DataLoader(val_data, batch_size=config.batch_size, shuffle=False)
model = Net(model_name).to(device)
criterion = nn.CrossEntropyLoss()
# criterion = FocalLoss(0.5)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
# optimizer = torch.optim.Adagrad(model.parameters(), lr=1e-3)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)
# config.model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
best_val_acc = 0
last_improved_epoch = 0
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path, '{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch: ', cur_epoch + 1)
cur_step = 0
for batch_x, batch_y in train_loader:
batch_x, batch_y = batch_x.to(device), batch_y.to(device)
optimizer.zero_grad()
probs = model(batch_x)
train_loss = criterion(probs, batch_y)
train_loss.backward()
optimizer.step()
cur_step += 1
if cur_step % config.train_print_step == 0:
train_acc = accuracy(probs, batch_y)
msg = 'the current step: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}'
print(msg.format(cur_step, len(train_loader), train_loss.item(), train_acc[0].item()))
val_loss, val_acc = evaluate(model, val_loader, criterion)
if val_acc >= best_val_acc:
best_val_acc = val_acc
torch.save(model.state_dict(), model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
# msg = 'the current epoch: {0}/{1}, train loss: {2:>5.2}, train acc: {3:>6.2%}, ' \
# 'val loss: {4:>5.2}, val acc: {5:>6.2%}, {6}'
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_acc,
end_time - start_time, improved_str))
scheduler.step()
if cur_epoch - last_improved_epoch > config.patience_epoch:
print("No optimization for a long time, auto-stopping...")
break
del model
gc.collect()
def train(train_data, val_data, fold_idx=None):
train_dataset = MyDataset(train_data, tokenizer)
val_dataset = MyDataset(val_data, tokenizer)
train_loader = DataLoader(train_dataset, batch_size=config.batch_size)
val_loader = DataLoader(val_dataset, batch_size=config.batch_size)
model = Model().to(config.device)
# optimizer = torch.optim.Adam(model.parameters(), lr=model_config.learning_rate)
# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
optimizer = Ranger(model.parameters(), lr=5e-5)
period = int(len(train_loader) / config.train_print_step)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=period,
eta_min=5e-9)
# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.train_print_step, eta_min=1e-9)
if fold_idx is None:
print('start')
model_save_path = os.path.join(config.model_path,
'{}.bin'.format(model_name))
else:
print('start fold: {}'.format(fold_idx + 1))
model_save_path = os.path.join(
config.model_path, '{}_fold{}.bin'.format(model_name, fold_idx))
best_val_score = 0
last_improved_epoch = 0
adjust_lr_num = 0
y_true_list = []
y_pred_list = []
for cur_epoch in range(config.epochs_num):
start_time = int(time.time())
model.train()
print('epoch:{}, step:{}'.format(cur_epoch + 1, len(train_loader)))
cur_step = 0
# for batch_x, batch_y in train_loader:
for inputs in train_loader:
tweet = inputs["tweet"]
selected_text = inputs["selected_text"]
sentiment = inputs["sentiment"]
ids = inputs["ids"]
attention_mask = inputs["attention_mask"]
token_type_ids = inputs["token_type_ids"]
targets_start_idx = inputs["start_idx"]
targets_end_idx = inputs["end_idx"]
offsets = inputs["offsets"]
# Move ids, masks, and targets to gpu while setting as torch.long
ids = ids.to(config.device, dtype=torch.long)
token_type_ids = token_type_ids.to(config.device, dtype=torch.long)
attention_mask = attention_mask.to(config.device, dtype=torch.long)
targets_start_idx = targets_start_idx.to(config.device,
dtype=torch.long)
targets_end_idx = targets_end_idx.to(config.device,
dtype=torch.long)
optimizer.zero_grad()
start_logits, end_logits = model(
input_ids=ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
)
loss = loss_fn(start_logits, end_logits, targets_start_idx,
targets_end_idx)
loss.backward()
optimizer.step()
cur_step += 1
# pred_start_idxs = torch.argmax(start_logits, dim=1).cpu().data.numpy()
# pred_end_idxs = torch.argmax(end_logits, dim=1).cpu().data.numpy()
# for i in range(len(tweet)):
# y_true_list.append((tweet[i], selected_text[i], sentiment[i], offsets[i]))
# y_pred_list.append((pred_start_idxs[i], pred_end_idxs[i]))
if cur_step % config.train_print_step == 0:
scheduler.step()
msg = 'the current step: {0}/{1}, cost: {2}s'
print(
msg.format(cur_step, len(train_loader),
int(time.time()) - start_time))
# train_score = get_score(y_true_list, y_pred_list)
# msg = 'the current step: {0}/{1}, train score: {2:>6.2%}'
# print(msg.format(cur_step, len(train_loader), train_score))
# y_true_list = []
# y_pred_list = []
val_loss, val_score = evaluate(model, val_loader)
if val_score >= best_val_score:
best_val_score = val_score
torch.save(model.state_dict(), model_save_path)
improved_str = '*'
last_improved_epoch = cur_epoch
else:
improved_str = ''
msg = 'the current epoch: {0}/{1}, val loss: {2:>5.2}, val acc: {3:>6.2%}, cost: {4}s {5}'
end_time = int(time.time())
print(
msg.format(cur_epoch + 1, config.epochs_num, val_loss, val_score,
end_time - start_time, improved_str))
if cur_epoch - last_improved_epoch >= config.patience_epoch:
if adjust_lr_num >= model_config.adjust_lr_num:
print("No optimization for a long time, auto stopping...")
break
print("No optimization for a long time, adjust lr...")
last_improved_epoch = cur_epoch # 加上，不然会连续更新的
adjust_lr_num += 1
del model
gc.collect()
if fold_idx is not None:
model_score[fold_idx] = best_val_score
