def train():
train_data, dev_data, test_data, vocab = get_train_dev_test_vocab()
model = CNNText(vocab_size=len(vocab), embedding_dim=50, output_size=20)
model = torch.load(load_path)
loss = CrossEntropyLoss(pred=Const.OUTPUT, target=Const.TARGET)
metrics = AccuracyMetric(pred=Const.OUTPUT, target=Const.TARGET)
'''
trainer = Trainer(model=model,
train_data=train_data,
dev_data=dev_data,
loss=loss,
metrics=metrics,
n_epochs=100,
save_path=checkpoint_path)
trainer.train()
'''
tester = Tester(test_data, model, metrics=AccuracyMetric())
tester.test()
def test_control_C():
# 用于测试 ControlC , 再两次训练时用 Control+C 进行退出，如果最后不显示 "Test failed!" 则通过测试
from fastNLP import ControlC, Callback
import time
line1 = "\n\n\n\n\n*************************"
line2 = "*************************\n\n\n\n\n"
class Wait(Callback):
def on_epoch_end(self):
time.sleep(5)
data_set, model = prepare_env()
print(line1 + "Test starts!" + line2)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=[Wait(), ControlC(False)], check_code_level=2)
trainer.train()
print(line1 + "Program goes on ..." + line2)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=[Wait(), ControlC(True)], check_code_level=2)
trainer.train()
print(line1 + "Test failed!" + line2)
def bilstm_text():
w = pickle.load(open("weight.bin", "rb"))
(vocab, train_data, dev_data, test_data) = read_data()
model_lstm = MyBLSTMText(class_num=4,
vocab_size=len(vocab),
dropout=0.5,
embed_weights=w)
loss = CrossEntropyLoss()
metrics = AccuracyMetric()
trainer = Trainer(model=model_lstm,
train_data=train_data,
dev_data=dev_data,
optimizer=Adam(lr=0.0015),
print_every=10,
use_tqdm=False,
device='cuda:0',
save_path="./lstm_model",
loss=loss,
metrics=metrics)
# callbacks=[EarlyStopCallback(10)])
trainer.train()
tester = Tester(test_data, model_lstm, metrics=AccuracyMetric())
tester.test()
def test_AccuracyMetric1(self):
# (1) only input, targets passed
pred_dict = {"pred": torch.zeros(4, 3)}
target_dict = {'target': torch.zeros(4)}
metric = AccuracyMetric()
metric(pred_dict=pred_dict, target_dict=target_dict)
print(metric.get_metric())
def test_AccuaryMetric7(self):
# (7) check map, match
metric = AccuracyMetric(pred='predictions', target='targets')
pred_dict = {"predictions": torch.randn(4, 3, 2)}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
res = metric.get_metric()
ans = (torch.argmax(pred_dict["predictions"], dim=2).float() == target_dict["targets"]).float().mean()
self.assertAlmostEqual(res["acc"], float(ans), places=4)
def test_AccuaryMetric5(self):
# (5) check reset
metric = AccuracyMetric()
pred_dict = {"pred": torch.randn(4, 3, 2)}
target_dict = {'target': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
res = metric.get_metric(reset=False)
ans = (torch.argmax(pred_dict["pred"], dim=2).float() == target_dict["target"]).float().mean()
self.assertAlmostEqual(res["acc"], float(ans), places=4)
def test_AccuaryMetric8(self):
try:
metric = AccuracyMetric(pred='predictions', target='targets')
pred_dict = {"prediction": torch.zeros(4, 3, 2)}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict, )
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_AccuaryMetric4(self):
# (5) check reset
metric = AccuracyMetric()
pred_dict = {"pred": torch.randn(4, 3, 2)}
target_dict = {'target': torch.ones(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
ans = torch.argmax(pred_dict["pred"], dim=2).to(target_dict["target"]) == target_dict["target"]
res = metric.get_metric()
self.assertTrue(isinstance(res, dict))
self.assertTrue("acc" in res)
self.assertAlmostEqual(res["acc"], float(ans.float().mean()), places=3)
def test_evaluate_callback(self):
data_set, model = prepare_env()
from fastNLP import Tester
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
evaluate_callback = EvaluateCallback(data_set, tester)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=evaluate_callback, check_code_level=2)
trainer.train()
def test_AccuaryMetric10(self):
# (10) check _fast_metric
try:
metric = AccuracyMetric()
pred_dict = {"predictions": torch.zeros(4, 3, 2), "seq_len": torch.ones(3) * 3}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_seq_len(self):
N = 256
seq_len = torch.zeros(N).long()
seq_len[0] = 2
pred = {'pred': torch.ones(N, 2)}
target = {'target': torch.ones(N, 2), 'seq_len': seq_len}
metric = AccuracyMetric()
metric(pred_dict=pred, target_dict=target)
self.assertDictEqual(metric.get_metric(), {'acc': 1.})
seq_len[1:] = 1
metric(pred_dict=pred, target_dict=target)
self.assertDictEqual(metric.get_metric(), {'acc': 1.})
def test_AccuaryMetric9(self):
# (9) check map, include unused
try:
metric = AccuracyMetric(pred='prediction', target='targets')
pred_dict = {"prediction": torch.zeros(4, 3, 2), 'unused': 1}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_fitlog_callback(self):
import fitlog
fitlog.set_log_dir(self.tempdir, new_log=True)
data_set, model = prepare_env()
from fastNLP import Tester
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
fitlog_callback = FitlogCallback(data_set, tester)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=fitlog_callback, check_code_level=2)
trainer.train()
def test_AccuracyMetric2(self):
# (2) with corrupted size
try:
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4)}
metric = AccuracyMetric()
metric(pred_dict=pred_dict, target_dict=target_dict, )
print(metric.get_metric())
except Exception as e:
print(e)
return
print("No exception catches.")
def test_case_1(self):
# 检查报错提示能否正确提醒用户
dataset = prepare_fake_dataset2('x1', 'x_unused')
dataset.rename_field('x_unused', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y', 'x1')
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = x1 + x2
time.sleep(0.1)
# loss = F.cross_entropy(x, y)
return {'preds': x}
model = Model()
with self.assertRaises(NameError):
tester = Tester(data=dataset,
model=model,
metrics=AccuracyMetric())
tester.test()
def test_collect_fn3(self):
"""
测试应该会覆盖
:return:
"""
dataset = prepare_fake_dataset2('x1', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y')
import torch
def fn(ins_list):
x = []
for ind, ins in ins_list:
x.append(ins['x1']+ins['x2'])
x = torch.FloatTensor(x)
return {'x1':torch.zeros_like(x)}, {'target':torch.zeros(x.size(0)).long(), 'y':x}
dataset.add_collect_fn(fn)
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 1, bias=False)
def forward(self, x1):
x1 = self.fc(x1)
assert x1.sum()==0, "Should be replaced to one"
# loss = F.cross_entropy(x, y)
return {'pred': x1}
model = Model()
trainer = Trainer(train_data=dataset, model=model, loss=CrossEntropyLoss(), print_every=2,
dev_data=dataset, metrics=AccuracyMetric(), use_tqdm=False, n_epochs=1)
best_metric = trainer.train()['best_eval']['AccuracyMetric']['acc']
self.assertTrue(best_metric==1)
def test_collect_fn2(self):
"""测试能否实现batch_x, batch_y"""
dataset = prepare_fake_dataset2('x1', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y', 'x1')
import torch
def fn(ins_list):
x = []
for ind, ins in ins_list:
x.append(ins['x1']+ins['x2'])
x = torch.FloatTensor(x)
return {'x':x}, {'target':x[:, :4].argmax(dim=-1)}
dataset.add_collect_fn(fn)
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2, x):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = self.fc(x)
sum_x = x1 + x2 + x
time.sleep(0.1)
# loss = F.cross_entropy(x, y)
return {'pred': sum_x}
model = Model()
trainer = Trainer(train_data=dataset, model=model, loss=CrossEntropyLoss(), print_every=2,
dev_data=dataset, metrics=AccuracyMetric(), use_tqdm=False)
trainer.train()
def test_trainer_suggestion6(self):
# 检查报错提示能否正确提醒用户
# 这里传入多余参数，让其duplicate
dataset = prepare_fake_dataset2('x1', 'x_unused')
dataset.rename_field('x_unused', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y', 'x1')
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = x1 + x2
time.sleep(0.1)
# loss = F.cross_entropy(x, y)
return {'preds': x}
model = Model()
with self.assertRaises(NameError):
trainer = Trainer(train_data=dataset,
model=model,
loss=CrossEntropyLoss(),
print_every=2,
dev_data=dataset,
metrics=AccuracyMetric(),
use_tqdm=False)
def test_gradient_clip(self):
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[GradientClipCallback(model.parameters(), clip_value=2)], check_code_level=2)
trainer.train()
def test_save_path(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
save_path = 'test_save_models'
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=save_path,
use_tqdm=True,
check_code_level=2)
trainer.train()
def test_readonly_property(self):
from fastNLP.core.callback import Callback
passed_epochs = []
total_epochs = 5
class MyCallback(Callback):
def __init__(self):
super(MyCallback, self).__init__()
def on_epoch_begin(self):
passed_epochs.append(self.epoch)
print(self.n_epochs, self.n_steps, self.batch_size)
print(self.model)
print(self.optimizer)
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=total_epochs,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[MyCallback()])
trainer.train()
assert passed_epochs == list(range(1, total_epochs + 1))
def run4(self):
set_rng_seed(100)
data_set, model = prepare_env()
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
trainer = DistTrainer(
train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size_per_gpu=32,
n_epochs=3,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=self.save_path,
)
trainer.train()
"""
# 应该正确运行
"""
if trainer.is_master and os.path.exists(self.save_path):
shutil.rmtree(self.save_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--methods",
"-m",
default="lstm",
choices=["rnn", "lstm", "cnn"])
parser.add_argument("--n_epochs", "-n", default=5, type=int)
parser.add_argument("--embedding", "-e", default=100, type=int)
parser.add_argument("--category", "-c", default=4, type=int)
parser.add_argument("--batch", "-b", default=4, type=int)
parser.add_argument("--learning_rate", "-l", default=0.005, type=float)
args = parser.parse_args()
if args.category > 20 or args.category < 1:
raise Exception("the number of category must be between 1 and 20")
train_data, test_data, dic_size = handle_data(args.category)
if args.methods == "rnn":
model = rnn(dic_size, args.category)
output = "rnn_model.pth"
elif args.methods == "lstm":
model = myLSTM(dic_size, args.category)
output = "lstm_model.pth"
else:
#model = cnn(dic_size, args.category)
model = torch.load("cnn_model.pth")
output = "cnn_model.pth"
trainer = Trainer(train_data,
model,
loss=CrossEntropyLoss(pred="pred", target='target'),
optimizer=SGD(model.parameters(), lr=args.learning_rate),
n_epochs=args.n_epochs,
dev_data=test_data,
metrics=AccuracyMetric(pred="pred", target='target'),
batch_size=args.batch)
trainer.train()
torch.save(model, output)
def test_trainer_data_parallel(self):
if torch.cuda.device_count() > 1:
from fastNLP import AccuracyMetric
dataset = prepare_fake_dataset2('x1', 'x2')
dataset.set_input('x1', 'x2', 'y', flag=True)
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2, y=None):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = x1 + x2
if self.training:
loss = F.cross_entropy(x, y)
return {'loss': loss}
else:
return {'pred': x, 'target': y}
model = Model()
trainer = Trainer(train_data=dataset,
model=model,
print_every=2,
use_tqdm=False,
dev_data=dataset,
metrics=AccuracyMetric(),
device=[0, 1])
trainer.train(load_best_model=False)
def test_early_stop(self):
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.01), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[EarlyStopCallback(5)], check_code_level=2)
trainer.train()
def test_AccuracyMetric3(self):
# (3) the second batch is corrupted size
try:
metric = AccuracyMetric()
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4)}
metric(pred_dict=pred_dict, target_dict=target_dict)
print(metric.get_metric())
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_warmup_callback(self):
data_set, model = prepare_env()
warmup_callback = WarmupCallback()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=warmup_callback, check_code_level=2)
trainer.train()
def test_CheckPointCallback(self):
from fastNLP import CheckPointCallback, Callback
from fastNLP import Tester
class RaiseCallback(Callback):
def __init__(self, stop_step=10):
super().__init__()
self.stop_step = stop_step
def on_backward_begin(self, loss):
if self.step > self.stop_step:
raise RuntimeError()
data_set, model = prepare_env()
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
import fitlog
fitlog.set_log_dir(self.tempdir, new_log=True)
tempfile_path = os.path.join(self.tempdir, 'chkt.pt')
callbacks = [CheckPointCallback(tempfile_path)]
fitlog_callback = FitlogCallback(data_set, tester)
callbacks.append(fitlog_callback)
callbacks.append(RaiseCallback(100))
try:
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=callbacks, check_code_level=2)
trainer.train()
except:
pass
# 用下面的代码模拟重新运行
data_set, model = prepare_env()
callbacks = [CheckPointCallback(tempfile_path)]
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
fitlog_callback = FitlogCallback(data_set, tester)
callbacks.append(fitlog_callback)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=callbacks, check_code_level=2)
trainer.train()
def __init__(self, masker, task_lst, vocabs, optimizer, args):
"""
:param model: 模型
:param description: 模型描述
:param task_lst: 任务列表
:param optimizer: 优化器
:param log_path: TensorboardX存储文件夹
:param save_path: 模型存储位置
:param accumulation_steps: 累积梯度
:param print_every: 评估间隔
"""
self.logger = fastNLP.logger
self.masker = masker
self.task_lst = task_lst
self.save_path = args.save_path
self.description = args.exp_name
self.optim = optimizer
self.vocabs = vocabs
n_steps = (int(
len(task_lst) * len(task_lst[0].train_set) * 100 / args.batch_size)
+ 1)
args.n_steps = n_steps
self.epoch_scheduler = get_scheduler(args, self.optim)
self.scheduler = None
self.logger.info('Using scheduler {}'.format(self.scheduler))
self.accumulation_steps = args.accumulation_steps
self.print_every = args.print_every
self.batch_size = args.batch_size
self.save_ep = args.save_ep
include_tasks = args.tasks
if include_tasks is None:
self.empty_tasks = set()
else:
self.empty_tasks = set(range(len(
self.task_lst))) - set(include_tasks)
self.steps = 0
self.best_acc = 0
self.best_epoch = 0
self.metrics = []
for t in task_lst:
if has_acc(t.task_name):
self.metrics.append(AccuracyMetric())
else:
self.metrics.append(
SpanFPreRecMetric(
self.vocabs[t.task_name],
encoding_type="bioes"
if t.task_name == "ner" else "bio",
))
# self.logger.info(self.metrics)
tb_path = "eval" if args.evaluate else "train"
self.summary_writer = SummaryWriter(os.path.join(
args.tb_path, tb_path))
def test_lr_scheduler(self):
data_set, model = prepare_env()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
trainer = Trainer(data_set, model, optimizer=optimizer, loss=BCELoss(pred="predict", target="y"), batch_size=32,
n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[LRScheduler(torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1))],
check_code_level=2)
trainer.train()
