def test_control_C():
# 用于测试 ControlC , 再两次训练时用 Control+C 进行退出，如果最后不显示 "Test failed!" 则通过测试
from fastNLP import ControlC, Callback
import time
line1 = "\n\n\n\n\n*************************"
line2 = "*************************\n\n\n\n\n"
class Wait(Callback):
def on_epoch_end(self):
time.sleep(5)
data_set, model = prepare_env()
print(line1 + "Test starts!" + line2)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=[Wait(), ControlC(False)], check_code_level=2)
trainer.train()
print(line1 + "Program goes on ..." + line2)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=[Wait(), ControlC(True)], check_code_level=2)
trainer.train()
print(line1 + "Test failed!" + line2)
def test_save_path(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
save_path = 'test_save_models'
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=save_path,
use_tqdm=True,
check_code_level=2)
trainer.train()
import os
import shutil
self.assertTrue(os.path.exists(save_path))
if os.path.exists(save_path):
shutil.rmtree(save_path)
# 无dev_data的训练
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=None,
metrics=None,
validate_every=-1,
save_path=save_path,
use_tqdm=True,
check_code_level=2)
trainer.train()
self.assertTrue(os.path.exists(save_path))
if os.path.exists(save_path):
shutil.rmtree(save_path)
def test_gradient_clip(self):
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[GradientClipCallback(model.parameters(), clip_value=2)], check_code_level=2)
trainer.train()
def run1(self):
# test distributed training
print('local rank', get_local_rank())
set_rng_seed(100)
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
model = NaiveClassifier(2, 2)
trainer = DistTrainer(
model=model,
train_data=data_set,
optimizer=SGD(lr=0.1),
loss=CrossEntropyLoss(pred="predict", target="y"),
batch_size_per_gpu=8,
n_epochs=3,
print_every=50,
save_path=self.save_path,
)
trainer.train()
"""
# 应该正确运行
"""
if trainer.is_master and os.path.exists(self.save_path):
shutil.rmtree(self.save_path)
def test_readonly_property(self):
from fastNLP.core.callback import Callback
passed_epochs = []
total_epochs = 5
class MyCallback(Callback):
def __init__(self):
super(MyCallback, self).__init__()
def on_epoch_begin(self):
passed_epochs.append(self.epoch)
print(self.n_epochs, self.n_steps, self.batch_size)
print(self.model)
print(self.optimizer)
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=total_epochs,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[MyCallback()])
trainer.train()
assert passed_epochs == list(range(1, total_epochs + 1))
def test_early_stop(self):
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.01), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=20, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[EarlyStopCallback(5)], check_code_level=2)
trainer.train()
def run4(self):
set_rng_seed(100)
data_set, model = prepare_env()
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
trainer = DistTrainer(
train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size_per_gpu=32,
n_epochs=3,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=self.save_path,
)
trainer.train()
"""
# 应该正确运行
"""
if trainer.is_master and os.path.exists(self.save_path):
shutil.rmtree(self.save_path)
def test_warmup_callback(self):
data_set, model = prepare_env()
warmup_callback = WarmupCallback()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=warmup_callback, check_code_level=2)
trainer.train()
def test_CheckPointCallback(self):
from fastNLP import CheckPointCallback, Callback
from fastNLP import Tester
class RaiseCallback(Callback):
def __init__(self, stop_step=10):
super().__init__()
self.stop_step = stop_step
def on_backward_begin(self, loss):
if self.step > self.stop_step:
raise RuntimeError()
data_set, model = prepare_env()
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
import fitlog
fitlog.set_log_dir(self.tempdir, new_log=True)
tempfile_path = os.path.join(self.tempdir, 'chkt.pt')
callbacks = [CheckPointCallback(tempfile_path)]
fitlog_callback = FitlogCallback(data_set, tester)
callbacks.append(fitlog_callback)
callbacks.append(RaiseCallback(100))
try:
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=callbacks, check_code_level=2)
trainer.train()
except:
pass
# 用下面的代码模拟重新运行
data_set, model = prepare_env()
callbacks = [CheckPointCallback(tempfile_path)]
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
fitlog_callback = FitlogCallback(data_set, tester)
callbacks.append(fitlog_callback)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=callbacks, check_code_level=2)
trainer.train()
def test_control_C_callback(self):
class Raise(Callback):
def on_epoch_end(self):
raise KeyboardInterrupt
flags = [False]
def set_flag():
flags[0] = not flags[0]
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size=32,
n_epochs=20,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
use_tqdm=True,
callbacks=[Raise(),
ControlC(False, set_flag)],
check_code_level=2)
trainer.train()
self.assertEqual(flags[0], False)
trainer = Trainer(data_set,
model,
optimizer=SGD(lr=0.1),
loss=BCELoss(pred="predict", target="y"),
batch_size=32,
n_epochs=20,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
use_tqdm=True,
callbacks=[Raise(),
ControlC(True, set_flag)],
check_code_level=2)
trainer.train()
self.assertEqual(flags[0], True)
def test_run_fp16(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier2(2, 1)
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=0)
trainer.train(load_best_model=False)
model = NaiveClassifier2(2, 1)
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=0,
test_use_fp16=False)
trainer.train(load_best_model=False)
def test_early_stop_callback(self):
"""
需要观察是否真的 EarlyStop
"""
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=2, n_epochs=10, print_every=5, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=EarlyStopCallback(1), check_code_level=2)
trainer.train()
def test_evaluate_callback(self):
data_set, model = prepare_env()
from fastNLP import Tester
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
evaluate_callback = EvaluateCallback(data_set, tester)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=evaluate_callback, check_code_level=2)
trainer.train()
def test_TensorboardCallback(self):
data_set, model = prepare_env()
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=False,
callbacks=[TensorboardCallback("loss", "metric")], check_code_level=2)
trainer.train()
import os
import shutil
path = os.path.join("./", 'tensorboard_logs_{}'.format(trainer.start_time))
if os.path.exists(path):
shutil.rmtree(path)
def test_save_model_callback(self):
data_set, model = prepare_env()
top = 3
save_model_callback = SaveModelCallback(self.tempdir, top=top)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=save_model_callback, check_code_level=2)
trainer.train()
timestamp = os.listdir(self.tempdir)[0]
self.assertEqual(len(os.listdir(os.path.join(self.tempdir, timestamp))), top)
def test_KeyBoardInterrupt(self):
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=5,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
callbacks=[ControlC(False)])
trainer.train()
def test_LRFinder(self):
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=5,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
callbacks=[LRFinder(len(data_set) // 32)])
trainer.train()
def test_fitlog_callback(self):
import fitlog
fitlog.set_log_dir(self.tempdir, new_log=True)
data_set, model = prepare_env()
from fastNLP import Tester
tester = Tester(data=data_set, model=model, metrics=AccuracyMetric(pred="predict", target="y"))
fitlog_callback = FitlogCallback(data_set, tester)
trainer = Trainer(data_set, model, optimizer=SGD(lr=0.1), loss=BCELoss(pred="predict", target="y"),
batch_size=32, n_epochs=5, print_every=50, dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"), use_tqdm=True,
callbacks=fitlog_callback, check_code_level=2)
trainer.train()
def test_TensorboardCallback(self):
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=5,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[TensorboardCallback("loss", "metric")])
trainer.train()
def train():
config = Config()
train_data, dev_data, vocabulary = get_dataset(config.data_path)
poetry_model = PoetryModel(vocabulary_size=len(vocabulary),
embedding_size=config.embedding_size,
hidden_size=config.hidden_size)
loss = Loss(pred='output', target='target')
perplexity = Perplexity(pred='output', target='target')
print("optimizer:", config.optimizer)
print("momentum:", config.momentum)
if config.optimizer == 'adam':
optimizer = Adam(lr=config.lr, weight_decay=config.weight_decay)
elif config.optimizer == 'sgd':
optimizer = SGD(lr=config.lr, momentum=config.momentum)
elif config.optimizer == 'adagrad':
optimizer = Adagrad(lr=config.lr, weight_decay=config.weight_decay)
elif config.optimizer == 'adadelta':
optimizer = Adadelta(lr=config.lr,
rho=config.rho,
eps=config.eps,
weight_decay=config.weight_decay)
timing = TimingCallback()
early_stop = EarlyStopCallback(config.patience)
trainer = Trainer(train_data=train_data,
model=poetry_model,
loss=loss,
metrics=perplexity,
n_epochs=config.epoch,
batch_size=config.batch_size,
print_every=config.print_every,
validate_every=config.validate_every,
dev_data=dev_data,
save_path=config.save_path,
optimizer=optimizer,
check_code_level=config.check_code_level,
metric_key="-PPL",
sampler=RandomSampler(),
prefetch=False,
use_tqdm=True,
device=config.device,
callbacks=[timing, early_stop])
trainer.train()
def test_case(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
trainer = Trainer(train_set, model,
loss=BCELoss(pred="predict", target="y"),
metrics=AccuracyMetric(pred="predict", target="y"),
n_epochs=10,
batch_size=32,
print_every=50,
validate_every=-1,
dev_data=dev_set,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=True,
save_path=None)
trainer.train()
"""
def train(config):
train_data = pickle.load(
open(os.path.join(config.data_path, config.train_name), "rb"))
# debug
train_data = train_data[0:100]
dev_data = pickle.load(
open(os.path.join(config.data_path, config.dev_name), "rb"))
print(len(train_data), len(dev_data))
# test_data = pickle.load(open(os.path.join(config.data_path, config.test_name), "rb"))
# load w2v data
# weight = pickle.load(open(os.path.join(config.data_path, config.weight_name), "rb"))
word_vocab = pickle.load(
open(os.path.join(config.data_path, config.word_vocab_name), "rb"))
char_vocab = pickle.load(
open(os.path.join(config.data_path, config.char_vocab_name), "rb"))
pos_vocab = pickle.load(
open(os.path.join(config.data_path, config.pos_vocab_name), "rb"))
spo_vocab = pickle.load(
open(os.path.join(config.data_path, config.spo_vocab_name), "rb"))
tag_vocab = pickle.load(
open(os.path.join(config.data_path, config.tag_vocab_name), "rb"))
print('word vocab', len(word_vocab))
print('char vocab', len(char_vocab))
print('pos vocab', len(pos_vocab))
print('spo vocab', len(spo_vocab))
print('tag vocab', len(tag_vocab))
model = BiLSTM_CRF(config.batch_size,
len(word_vocab),
len(char_vocab),
len(pos_vocab),
len(spo_vocab),
config.embed_dim,
config.hidden_dim,
tag_vocab.idx2word,
dropout=0.5)
optimizer = SGD(lr=config.lr, momentum=config.momentum)
timing = TimingCallback()
early_stop = EarlyStopCallback(config.patience)
loss = NLLLoss()
metrics = SpanFPreRecMetric(tag_vocab)
# accuracy = AccuracyMetric(pred='output', target='target')
trainer = Trainer(train_data=train_data,
model=model,
loss=loss,
metrics=metrics,
batch_size=config.batch_size,
n_epochs=config.epoch,
dev_data=dev_data,
save_path=config.save_path,
check_code_level=-1,
print_every=100,
validate_every=0,
optimizer=optimizer,
use_tqdm=False,
device=config.device,
callbacks=[timing, early_stop])
trainer.train()
def test_run_data_parallel(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
class NaiveClassifier2(BaseModel):
r"""
一个简单的分类器例子，可用于各种测试
"""
def __init__(self, in_feature_dim, out_feature_dim):
super(NaiveClassifier2, self).__init__()
self.mlp = MLP(
[in_feature_dim, in_feature_dim, out_feature_dim])
def forward(self, x):
return {"predict": self.mlp(x)}
def predict(self, x):
return {"predict": torch.sigmoid(self.mlp(x)) > 0.5}
model = NaiveClassifier2(2, 1)
with self.assertRaises(RuntimeError):
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict",
target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=[0, 1])
with self.assertRaises(RuntimeError):
class NaiveClassifier3(BaseModel):
r"""
一个简单的分类器例子，可用于各种测试
"""
def __init__(self, in_feature_dim, out_feature_dim):
super(NaiveClassifier3, self).__init__()
self.mlp = MLP(
[in_feature_dim, in_feature_dim, out_feature_dim])
@torch.cuda.amp.autocast()
def forward(self, x):
return {"predict": self.mlp(x)}
@torch.cuda.amp.autocast()
def predict(self, x):
return {"predict": torch.sigmoid(self.mlp(x)) > 0.5}
model = NaiveClassifier3(2, 1)
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict",
target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=[0, 1],
test_use_fp16=True)
class NaiveClassifier4(BaseModel):
r"""
一个简单的分类器例子，可用于各种测试
"""
def __init__(self, in_feature_dim, out_feature_dim):
super(NaiveClassifier4, self).__init__()
self.mlp = MLP(
[in_feature_dim, in_feature_dim, out_feature_dim])
def forward(self, x):
with torch.cuda.amp.autocast():
return {"predict": self.mlp(x)}
def predict(self, x):
with torch.cuda.amp.autocast():
return {"predict": torch.sigmoid(self.mlp(x)) > 0.5}
model = NaiveClassifier4(2, 1)
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict", target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=[0, 1],
test_use_fp16=True)
trainer.train(load_best_model=False)
def test_raise_error(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier2(2, 1)
with self.assertRaises(RuntimeError):
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict",
target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True)
with self.assertRaises(RuntimeError):
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict",
target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device='cpu')
with self.assertRaises(RuntimeError):
trainer = Trainer(train_set,
model,
optimizer=SGD(lr=0.1),
loss=BCEWithLogits(pred="predict", target="y"),
batch_size=32,
n_epochs=10,
print_every=50,
dev_data=dev_set,
metrics=AccuracyMetric(pred="predict",
target="y"),
validate_every=-1,
save_path=None,
use_tqdm=True,
check_code_level=2,
fp16=True,
device=torch.device('cpu'))
def test_SGD(self):
optim = SGD(model_params=torch.nn.Linear(10, 3).parameters())
self.assertTrue("lr" in optim.__dict__["settings"])
self.assertTrue("momentum" in optim.__dict__["settings"])
res = optim.construct_from_pytorch(torch.nn.Linear(10, 3).parameters())
self.assertTrue(isinstance(res, torch.optim.SGD))
optim = SGD(lr=0.001)
self.assertEqual(optim.__dict__["settings"]["lr"], 0.001)
res = optim.construct_from_pytorch(torch.nn.Linear(10, 3).parameters())
self.assertTrue(isinstance(res, torch.optim.SGD))
optim = SGD(lr=0.002, momentum=0.989)
self.assertEqual(optim.__dict__["settings"]["lr"], 0.002)
self.assertEqual(optim.__dict__["settings"]["momentum"], 0.989)
optim = SGD(0.001)
self.assertEqual(optim.__dict__["settings"]["lr"], 0.001)
res = optim.construct_from_pytorch(torch.nn.Linear(10, 3).parameters())
self.assertTrue(isinstance(res, torch.optim.SGD))
with self.assertRaises(TypeError):
_ = SGD("???")
with self.assertRaises(TypeError):
_ = SGD(0.001, lr=0.002)
