def init_model():
train_data, dev_data, test_data, vocab = readdata()
model = CNN((len(vocab), 128),
num_classes=target_len,
padding=2,
dropout=0.1)
#model = torch.load("rnnmodel/best_RNN_accuracy_2019-05-22-17-18-46")
trainer = Trainer(model=model,
train_data=train_data,
dev_data=dev_data,
device=0,
save_path='cnnmodel',
loss=loss,
metrics=metrics,
callbacks=[FitlogCallback(test_data)])
tester = Tester(test_data, model, metrics=AccuracyMetric())
print(2)
model2 = RNN(embed_num=len(vocab),
input_size=256,
hidden_size=256,
target_size=target_len)
#model2 = torch.load("rnnmodel/best_RNN_accuracy_2019-05-22-17-18-46")
trainer2 = Trainer(model=model2,
train_data=train_data,
dev_data=dev_data,
loss=loss,
metrics=metrics,
save_path='rnnmodel',
batch_size=32,
n_epochs=20,
device=0)
tester2 = Tester(test_data, model, metrics=AccuracyMetric())
return trainer, trainer2, tester, tester2
def test_AccuaryMetric7(self):
# (7) check map, match
metric = AccuracyMetric(pred='predictions', target='targets')
pred_dict = {"predictions": torch.zeros(4, 3, 2)}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 1})
def test_AccuracyMetric1(self):
# (1) only input, targets passed
pred_dict = {"pred": torch.zeros(4, 3)}
target_dict = {'target': torch.zeros(4)}
metric = AccuracyMetric()
metric(pred_dict=pred_dict, target_dict=target_dict, )
print(metric.get_metric())
def test_AccuaryMetric7(self):
# (7) check map, match
metric = AccuracyMetric(pred='predictions', target='targets')
pred_dict = {"predictions": torch.randn(4, 3, 2)}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
res = metric.get_metric()
ans = (torch.argmax(
pred_dict["predictions"],
dim=2).float() == target_dict["targets"]).float().mean()
self.assertAlmostEqual(res["acc"], float(ans), places=4)
def test_AccuaryMetric5(self):
# (5) check reset
metric = AccuracyMetric()
pred_dict = {"pred": torch.randn(4, 3, 2)}
target_dict = {'target': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
res = metric.get_metric(reset=False)
ans = (torch.argmax(
pred_dict["pred"],
dim=2).float() == target_dict["target"]).float().mean()
self.assertAlmostEqual(res["acc"], float(ans), places=4)
def test_AccuaryMetric10(self):
# (10) check _fast_metric
try:
metric = AccuracyMetric()
pred_dict = {"predictions": torch.zeros(4, 3, 2), "masks": torch.zeros(4, 3)}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_AccuaryMetric5(self):
# (5) check reset
metric = AccuracyMetric()
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(reset=False), {'acc': 1})
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4, 3) + 1}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 0.5})
def test_AccuaryMetric4(self):
# (5) check reset
metric = AccuracyMetric()
pred_dict = {"pred": torch.randn(4, 3, 2)}
target_dict = {'target': torch.ones(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
ans = torch.argmax(pred_dict["pred"], dim=2).to(
target_dict["target"]) == target_dict["target"]
res = metric.get_metric()
self.assertTrue(isinstance(res, dict))
self.assertTrue("acc" in res)
self.assertAlmostEqual(res["acc"], float(ans.float().mean()), places=3)
def test_AccuaryMetric9(self):
# (9) check map, include unused
try:
metric = AccuracyMetric(pred='prediction', target='targets')
pred_dict = {"prediction": torch.zeros(4, 3, 2), 'unused': 1}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict)
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_AccuaryMetric8(self):
# (8) check map, does not match. use stop_fast_param to stop fast param map
try:
metric = AccuracyMetric(pred='predictions', target='targets')
pred_dict = {"prediction": torch.zeros(4, 3, 2), "stop_fast_param": 1}
target_dict = {'targets': torch.zeros(4, 3)}
metric(pred_dict=pred_dict, target_dict=target_dict, )
self.assertDictEqual(metric.get_metric(), {'acc': 1})
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_AccuracyMetric2(self):
# (2) with corrupted size
try:
pred_dict = {"pred": torch.zeros(4, 3, 2)}
target_dict = {'target': torch.zeros(4)}
metric = AccuracyMetric()
metric(pred_dict=pred_dict, target_dict=target_dict, )
print(metric.get_metric())
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_trainer_suggestion6(self):
# 检查报错提示能否正确提醒用户
# 这里传入多余参数，让其duplicate
dataset = prepare_fake_dataset2('x1', 'x_unused')
dataset.rename_field('x_unused', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y', 'x1')
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = x1 + x2
time.sleep(0.1)
# loss = F.cross_entropy(x, y)
return {'preds': x}
model = Model()
with self.assertRaises(NameError):
trainer = Trainer(train_data=dataset,
model=model,
dev_data=dataset,
loss=CrossEntropyLoss(),
metrics=AccuracyMetric(),
use_tqdm=False,
print_every=2)
def test_case_1(self):
# 检查报错提示能否正确提醒用户
dataset = prepare_fake_dataset2('x1', 'x_unused')
dataset.rename_field('x_unused', 'x2')
dataset.set_input('x1', 'x2')
dataset.set_target('y', 'x1')
class Model(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(5, 4)
def forward(self, x1, x2):
x1 = self.fc(x1)
x2 = self.fc(x2)
x = x1 + x2
time.sleep(0.1)
# loss = F.cross_entropy(x, y)
return {'preds': x}
model = Model()
with self.assertRaises(NameError):
tester = Tester(data=dataset,
model=model,
metrics=AccuracyMetric())
tester.test()
def test_model(data_test, model):
# 使用tester来进行测试
tester = Tester(data=data_test,
model=model,
metrics=AccuracyMetric(pred="predict", target="label_seq"),
batch_size=4)
acc = tester.test()
print(acc)
def test_fastnlp_1min_tutorial(self):
# tutorials/fastnlp_1min_tutorial.ipynb
data_path = "test/data_for_tests/tutorial_sample_dataset.csv"
ds = DataSet.read_csv(data_path,
headers=('raw_sentence', 'label'),
sep='\t')
print(ds[1])
# 将所有数字转为小写
ds.apply(lambda x: x['raw_sentence'].lower(),
new_field_name='raw_sentence')
# label转int
ds.apply(lambda x: int(x['label']),
new_field_name='target',
is_target=True)
def split_sent(ins):
return ins['raw_sentence'].split()
ds.apply(split_sent, new_field_name='words', is_input=True)
# 分割训练集/验证集
train_data, dev_data = ds.split(0.3)
print("Train size: ", len(train_data))
print("Test size: ", len(dev_data))
from fastNLP import Vocabulary
vocab = Vocabulary(min_freq=2)
train_data.apply(lambda x: [vocab.add(word) for word in x['words']])
# index句子, Vocabulary.to_index(word)
train_data.apply(
lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words',
is_input=True)
dev_data.apply(lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words',
is_input=True)
from fastNLP.models import CNNText
model = CNNText((len(vocab), 50),
num_classes=5,
padding=2,
dropout=0.1)
from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric, Adam
trainer = Trainer(model=model,
train_data=train_data,
dev_data=dev_data,
loss=CrossEntropyLoss(),
optimizer=Adam(),
metrics=AccuracyMetric(target='target'))
trainer.train()
print('Train finished!')
def test_AccuaryMetric6(self):
# (6) check numpy array is not acceptable
try:
metric = AccuracyMetric()
pred_dict = {"pred": np.zeros((4, 3, 2))}
target_dict = {'target': np.zeros((4, 3))}
metric(pred_dict=pred_dict, target_dict=target_dict)
except Exception as e:
print(e)
return
self.assertTrue(True, False), "No exception catches."
def test_early_stop(self):
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=20,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.01),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[EarlyStopCallback(5)])
trainer.train()
def test_TensorboardCallback(self):
data_set, model = prepare_env()
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=5,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[TensorboardCallback("loss", "metric")])
trainer.train()
def test_gradient_clip(self):
data_set, model = prepare_env()
trainer = Trainer(
data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=20,
batch_size=32,
print_every=50,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[GradientClipCallback(model.parameters(), clip_value=2)])
trainer.train()
def train(epochs=10, lr=0.001):
global model
for i in range(epochs):
print('----------------- ', str(i + 1), ' ------------------')
trainer = Trainer(model=model,
train_data=train_set,
dev_data=test_set,
loss=CrossEntropyLoss(pred='output',
target='target'),
metrics=AccuracyMetric(pred='pred', target='target'),
optimizer=Adam(lr=lr),
save_path=None,
batch_size=1,
n_epochs=1)
trainer.train()
model.load_state_dict(copy.deepcopy(trainer.model.state_dict()))
# save('../model/cnn-' + str(kernel_sizes) + '-' + str(keep_proba) + '-' + str(i+1))
save('../model/lstm-' + str(input_dim) + '-' + str(hidden_dim) + '-' +
str(i + 1))
def test_lr_scheduler(self):
data_set, model = prepare_env()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
trainer = Trainer(data_set,
model,
loss=BCELoss(pred="predict", target="y"),
n_epochs=5,
batch_size=32,
print_every=50,
optimizer=optimizer,
check_code_level=2,
use_tqdm=False,
dev_data=data_set,
metrics=AccuracyMetric(pred="predict", target="y"),
callbacks=[
LRScheduler(
torch.optim.lr_scheduler.StepLR(optimizer,
step_size=10,
gamma=0.1))
])
trainer.train()
def test_case(self):
data_set = prepare_fake_dataset()
data_set.set_input("x", flag=True)
data_set.set_target("y", flag=True)
train_set, dev_set = data_set.split(0.3)
model = NaiveClassifier(2, 1)
trainer = Trainer(train_set,
model,
loss=BCELoss(pred="predict", target="y"),
metrics=AccuracyMetric(pred="predict", target="y"),
n_epochs=10,
batch_size=32,
print_every=50,
validate_every=-1,
dev_data=dev_set,
optimizer=SGD(lr=0.1),
check_code_level=2,
use_tqdm=True,
save_path=None)
trainer.train()
"""
import os
import torch
from fastNLP import Tester
from fastNLP.core.metrics import AccuracyMetric
from model import CNN,RNN
import pickle
import config
def load_model(model, save_path, model_name):
model_path = os.path.join(save_path, model_name)
states = torch.load(model_path).state_dict()
model.load_state_dict(states)
if __name__ == "__main__":
vocab = pickle.load(open(config.vocab_path, 'rb'))
test_data = pickle.load(open(config.test_data_path, 'rb'))
if config.model == "CNN":
model = CNN(len(vocab), config.intput_size, config.class_num)
elif config.model == "RNN":
model = RNN(len(vocab), config.intput_size, config.hidden_size, config.class_num,config.rnn_type)
load_model(model,
save_path=config.save_path,
model_name="RNNmax")
metrics = AccuracyMetric(pred="output", target="target")
tester = Tester(test_data, model, metrics=metrics, device='cuda:0')
tester.test()
nn.init.xavier_normal_(p)
except:
print_info(n)
exit(1208)
print_info('{}init pram{}'.format('*' * 15, '*' * 15))
loss = LossInForward()
encoding_type = 'bmeso'
f1_metric = SpanFPreRecMetric(vocabs['label'],
pred='pred',
target='target',
seq_len='seq_len',
encoding_type=encoding_type)
acc_metric = AccuracyMetric(
pred='pred',
target='target',
seq_len='seq_len',
)
acc_metric.set_metric_name('label_acc')
metrics = [f1_metric, acc_metric]
if args.self_supervised:
chars_acc_metric = AccuracyMetric(pred='chars_pred',
target='chars_target',
seq_len='seq_len')
chars_acc_metric.set_metric_name('chars_acc')
metrics.append(chars_acc_metric)
if args.see_param:
for n, p in model.named_parameters():
print_info('{}:{}'.format(n, p.size()))
print_info('see_param mode: finish')
def test_fastnlp_10min_tutorial(self):
# 从csv读取数据到DataSet
sample_path = "test/data_for_tests/tutorial_sample_dataset.csv"
dataset = CSVLoader(headers=['raw_sentence', 'label'], sep=' ')._load(sample_path)
print(len(dataset))
print(dataset[0])
print(dataset[-3])
dataset.append(Instance(raw_sentence='fake data', label='0'))
# 将所有数字转为小写
dataset.apply(lambda x: x['raw_sentence'].lower(), new_field_name='raw_sentence')
# label转int
dataset.apply(lambda x: int(x['label']), new_field_name='label')
# 使用空格分割句子
def split_sent(ins):
return ins['raw_sentence'].split()
dataset.apply(split_sent, new_field_name='words')
# 增加长度信息
dataset.apply(lambda x: len(x['words']), new_field_name='seq_len')
print(len(dataset))
print(dataset[0])
# DataSet.drop(func)筛除数据
dataset.drop(lambda x: x['seq_len'] <= 3, inplace=True)
print(len(dataset))
# 设置DataSet中，哪些field要转为tensor
# set target，loss或evaluate中的golden，计算loss，模型评估时使用
dataset.set_target("label")
# set input，模型forward时使用
dataset.set_input("words", "seq_len")
# 分出测试集、训练集
test_data, train_data = dataset.split(0.5)
print(len(test_data))
print(len(train_data))
# 构建词表, Vocabulary.add(word)
vocab = Vocabulary(min_freq=2)
train_data.apply(lambda x: [vocab.add(word) for word in x['words']])
vocab.build_vocab()
# index句子, Vocabulary.to_index(word)
train_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words')
test_data.apply(lambda x: [vocab.to_index(word) for word in x['words']], new_field_name='words')
print(test_data[0])
# 如果你们需要做强化学习或者GAN之类的项目，你们也可以使用这些数据预处理的工具
from fastNLP.core.batch import DataSetIter
from fastNLP.core.sampler import RandomSampler
batch_iterator = DataSetIter(dataset=train_data, batch_size=2, sampler=RandomSampler())
for batch_x, batch_y in batch_iterator:
print("batch_x has: ", batch_x)
print("batch_y has: ", batch_y)
break
from fastNLP.models import CNNText
model = CNNText((len(vocab), 50), num_classes=5, dropout=0.1)
from fastNLP import Trainer
from copy import deepcopy
# 更改DataSet中对应field的名称，要以模型的forward等参数名一致
train_data.rename_field('label', 'label_seq')
test_data.rename_field('label', 'label_seq')
loss = CrossEntropyLoss(target="label_seq")
metric = AccuracyMetric(target="label_seq")
# 实例化Trainer，传入模型和数据，进行训练
# 先在test_data拟合（确保模型的实现是正确的）
copy_model = deepcopy(model)
overfit_trainer = Trainer(train_data=test_data, model=copy_model, loss=loss, batch_size=32, n_epochs=5,
dev_data=test_data, metrics=metric, save_path=None)
overfit_trainer.train()
# 用train_data训练，在test_data验证
trainer = Trainer(model=model, train_data=train_data, dev_data=test_data,
loss=CrossEntropyLoss(target="label_seq"),
metrics=AccuracyMetric(target="label_seq"),
save_path=None,
batch_size=32,
n_epochs=5)
trainer.train()
print('Train finished!')
# 调用Tester在test_data上评价效果
from fastNLP import Tester
tester = Tester(data=test_data, model=model, metrics=AccuracyMetric(target="label_seq"),
batch_size=4)
acc = tester.test()
print(acc)
def train(args):
text_data = TextData()
with open(os.path.join(args.vocab_dir, args.vocab_data), 'rb') as fin:
text_data = pickle.load(fin)
vocab_size = text_data.vocab_size
class_num = text_data.class_num
# class_num = 1
seq_len = text_data.max_seq_len
print("(vocab_size,class_num,seq_len):({0},{1},{2})".format(
vocab_size, class_num, seq_len))
train_data = text_data.train_set
val_data = text_data.val_set
test_data = text_data.test_set
train_data.set_input('words', 'seq_len')
train_data.set_target('target')
val_data.set_input('words', 'seq_len')
val_data.set_target('target')
test_data.set_input('words', 'seq_len')
test_data.set_target('target')
init_embeds = None
if args.pretrain_model == "None":
print("No pretrained model with be used.")
print("vocabsize:{0}".format(vocab_size))
init_embeds = (vocab_size, args.embed_size)
elif args.pretrain_model == "word2vec":
embeds_path = os.path.join(args.prepare_dir, 'w2v_embeds.pkl')
print("Loading Word2Vec pretrained embedding from {0}.".format(
embeds_path))
with open(embeds_path, 'rb') as fin:
init_embeds = pickle.load(fin)
elif args.pretrain_model == 'glove':
embeds_path = os.path.join(args.prepare_dir, 'glove_embeds.pkl')
print(
"Loading Glove pretrained embedding from {0}.".format(embeds_path))
with open(embeds_path, 'rb') as fin:
init_embeds = pickle.load(fin)
elif args.pretrain_model == 'glove2wv':
embeds_path = os.path.join(args.prepare_dir, 'glove2wv_embeds.pkl')
print(
"Loading Glove pretrained embedding from {0}.".format(embeds_path))
with open(embeds_path, 'rb') as fin:
init_embeds = pickle.load(fin)
else:
init_embeds = (vocab_size, args.embed_size)
if args.model == "CNNText":
print("Using CNN Model.")
model = CNNText(init_embeds,
num_classes=class_num,
padding=2,
dropout=args.dropout)
elif args.model == "StarTransformer":
print("Using StarTransformer Model.")
model = STSeqCls(init_embeds,
num_cls=class_num,
hidden_size=args.hidden_size)
elif args.model == "MyCNNText":
model = MyCNNText(init_embeds=init_embeds,
num_classes=class_num,
padding=2,
dropout=args.dropout)
print("Using user defined CNNText")
elif args.model == "LSTMText":
print("Using LSTM Model.")
model = LSTMText(init_embeds=init_embeds,
output_dim=class_num,
hidden_dim=args.hidden_size,
num_layers=args.num_layers,
dropout=args.dropout)
elif args.model == "Bert":
print("Using Bert Model.")
else:
print("Using default model: CNNText.")
model = CNNText((vocab_size, args.embed_size),
num_classes=class_num,
padding=2,
dropout=0.1)
print(model)
if args.cuda:
device = torch.device('cuda')
else:
device = None
print("train_size:{0} ; val_size:{1} ; test_size:{2}".format(
train_data.get_length(), val_data.get_length(),
test_data.get_length()))
if args.optim == "Adam":
print("Using Adam as optimizer.")
optimizer = fastnlp_optim.Adam(lr=0.001,
weight_decay=args.weight_decay)
if (args.model_suffix == "default"):
args.model_suffix == args.optim
else:
print("No Optimizer will be used.")
optimizer = None
criterion = CrossEntropyLoss()
metric = AccuracyMetric()
model_save_path = os.path.join(args.model_dir, args.model,
args.model_suffix)
earlystop = EarlyStopCallback(args.patience)
fitlog_back = FitlogCallback({"val": val_data, "train": train_data})
trainer = Trainer(train_data=train_data,
model=model,
save_path=model_save_path,
device=device,
n_epochs=args.epochs,
optimizer=optimizer,
dev_data=val_data,
loss=criterion,
batch_size=args.batch_size,
metrics=metric,
callbacks=[fitlog_back, earlystop])
trainer.train()
print("Train Done.")
tester = Tester(data=val_data,
model=model,
metrics=metric,
batch_size=args.batch_size,
device=device)
tester.test()
print("Test Done.")
print("Predict the answer with best model...")
acc = 0.0
output = []
data_iterator = Batch(test_data, batch_size=args.batch_size)
for data_x, batch_y in data_iterator:
i_data = Variable(data_x['words']).cuda()
pred = model(i_data)[C.OUTPUT]
pred = pred.sigmoid()
# print(pred.shape)
output.append(pred.cpu().data)
output = torch.cat(output, 0).numpy()
print(output.shape)
print("Predict Done. {} records".format(len(output)))
result_save_path = os.path.join(args.result_dir,
args.model + "_" + args.model_suffix)
with open(result_save_path + ".pkl", 'wb') as f:
pickle.dump(output, f)
output = output.squeeze()[:, 1].tolist()
projectid = text_data.test_projectid.values
answers = []
count = 0
for i in range(len(output)):
if output[i] > 0.5:
count += 1
print("true sample count:{}".format(count))
add_count = 0
for i in range(len(projectid) - len(output)):
output.append([0.13])
add_count += 1
print("Add {} default result in predict.".format(add_count))
df = pd.DataFrame()
df['projectid'] = projectid
df['y'] = output
df.to_csv(result_save_path + ".csv", index=False)
print("Predict Done, results saved to {}".format(result_save_path))
fitlog.finish()
from fastNLP import Const
from fastNLP import AccuracyMetric
from fastNLP import CrossEntropyLoss
from fastNLP import BucketSampler
from fastNLP import Batch
import torch
import time
import fitlog
from fastNLP.core.callback import FitlogCallback
from fastNLP import Tester
from fastNLP import Callback
fitlog.commit('__file__') # auto commit your codes
fitlog.add_hyper_in_file ('__file__') # record your hyperparameters
loss = CrossEntropyLoss(pred=Const.OUTPUT, target=Const.TARGET)
metrics=AccuracyMetric(pred=Const.OUTPUT, target=Const.TARGET)
target_len = 20
def readdata():
global target_len
min_count = 10
#categories = ['comp.os.ms-windows.misc', 'rec.motorcycles', 'sci.space', 'talk.politics.misc', ]
dataset_train = fetch_20newsgroups(subset='train', data_home='../../..')
dataset_test = fetch_20newsgroups(subset='test', data_home='../../..')
data = dataset_train.data
target = dataset_train.target
target_len = len(dataset_train.target_names)
train_data = DataSet()
padding = 0
for i in range(len(data)):
data_t = re.sub("\d+|\s+|/", " ", data[i] )
from fastNLP.core.losses import CrossEntropyLoss
from fastNLP.core.metrics import AccuracyMetric
from fastNLP.core.optimizer import Adam
# from fastNLP.core.utils import save_pickle
from fastNLP.io.model_io import ModelSaver
# load model
model = DPCNN(max_features=len(vocab) + 1,
word_embedding_dimension=word_embedding_dimension,
max_sentence_length=max_sentence_length,
num_classes=num_classes,
weight=weight)
# define loss and metric
loss = CrossEntropyLoss(pred="output", target="label_seq")
metric = AccuracyMetric(pred="predict", target="label_seq")
# train model with train_data,and val model with test_data
# embedding=300 gaussian init，weight_decay=0.0001, lr=0.001，epoch=5
trainer = Trainer(model=model,
train_data=dataset_train,
dev_data=dataset_test,
loss=loss,
metrics=metric,
save_path='new_model.pkl',
batch_size=64,
n_epochs=5,
optimizer=Adam(lr=0.001, weight_decay=0.0001))
trainer.train()
# save pickle
def testENAS(self):
# 从csv读取数据到DataSet
sample_path = "tutorials/sample_data/tutorial_sample_dataset.csv"
dataset = DataSet.read_csv(sample_path,
headers=('raw_sentence', 'label'),
sep='\t')
print(len(dataset))
print(dataset[0])
print(dataset[-3])
dataset.append(Instance(raw_sentence='fake data', label='0'))
# 将所有数字转为小写
dataset.apply(lambda x: x['raw_sentence'].lower(),
new_field_name='raw_sentence')
# label转int
dataset.apply(lambda x: int(x['label']), new_field_name='label')
# 使用空格分割句子
def split_sent(ins):
return ins['raw_sentence'].split()
dataset.apply(split_sent, new_field_name='words')
# 增加长度信息
dataset.apply(lambda x: len(x['words']), new_field_name='seq_len')
print(len(dataset))
print(dataset[0])
# DataSet.drop(func)筛除数据
dataset.drop(lambda x: x['seq_len'] <= 3)
print(len(dataset))
# 设置DataSet中，哪些field要转为tensor
# set target，loss或evaluate中的golden，计算loss，模型评估时使用
dataset.set_target("label")
# set input，模型forward时使用
dataset.set_input("words", "seq_len")
# 分出测试集、训练集
test_data, train_data = dataset.split(0.5)
print(len(test_data))
print(len(train_data))
# 构建词表, Vocabulary.add(word)
vocab = Vocabulary(min_freq=2)
train_data.apply(lambda x: [vocab.add(word) for word in x['words']])
vocab.build_vocab()
# index句子, Vocabulary.to_index(word)
train_data.apply(
lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words')
test_data.apply(
lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words')
print(test_data[0])
# 如果你们需要做强化学习或者GAN之类的项目，你们也可以使用这些数据预处理的工具
from fastNLP.core.batch import Batch
from fastNLP.core.sampler import RandomSampler
batch_iterator = Batch(dataset=train_data,
batch_size=2,
sampler=RandomSampler())
for batch_x, batch_y in batch_iterator:
print("batch_x has: ", batch_x)
print("batch_y has: ", batch_y)
break
from fastNLP.models.enas_model import ENASModel
from fastNLP.models.enas_controller import Controller
model = ENASModel(embed_num=len(vocab), num_classes=5)
controller = Controller()
from fastNLP.models.enas_trainer import ENASTrainer
from copy import deepcopy
# 更改DataSet中对应field的名称，要以模型的forward等参数名一致
train_data.rename_field('words',
'word_seq') # input field 与 forward 参数一致
train_data.rename_field('label', 'label_seq')
test_data.rename_field('words', 'word_seq')
test_data.rename_field('label', 'label_seq')
loss = CrossEntropyLoss(pred="output", target="label_seq")
metric = AccuracyMetric(pred="predict", target="label_seq")
trainer = ENASTrainer(model=model,
controller=controller,
train_data=train_data,
dev_data=test_data,
loss=CrossEntropyLoss(pred="output",
target="label_seq"),
metrics=AccuracyMetric(pred="predict",
target="label_seq"),
check_code_level=-1,
save_path=None,
batch_size=32,
print_every=1,
n_epochs=3,
final_epochs=1)
trainer.train()
print('Train finished!')
# 调用Tester在test_data上评价效果
from fastNLP import Tester
tester = Tester(data=test_data,
model=model,
metrics=AccuracyMetric(pred="predict",
target="label_seq"),
batch_size=4)
acc = tester.test()
print(acc)
def test_tutorial(self):
# 从csv读取数据到DataSet
sample_path = "./data_for_tests/tutorial_sample_dataset.csv"
dataset = DataSet.read_csv(sample_path,
headers=('raw_sentence', 'label'),
sep='\t')
print(len(dataset))
print(dataset[0])
dataset.append(Instance(raw_sentence='fake data', label='0'))
dataset.apply(lambda x: x['raw_sentence'].lower(),
new_field_name='raw_sentence')
# label转int
dataset.apply(lambda x: int(x['label']), new_field_name='label')
# 使用空格分割句子
def split_sent(ins):
return ins['raw_sentence'].split()
dataset.apply(split_sent, new_field_name='words')
# 增加长度信息
dataset.apply(lambda x: len(x['words']), new_field_name='seq_len')
# print(len(dataset))
# print(dataset[0])
# DataSet.drop(func)筛除数据
dataset.drop(lambda x: x['seq_len'] <= 3)
print(len(dataset))
# 设置DataSet中，哪些field要转为tensor
# set target，loss或evaluate中的golden，计算loss，模型评估时使用
dataset.set_target("label")
# set input，模型forward时使用
dataset.set_input("words")
# 分出测试集、训练集
test_data, train_data = dataset.split(0.5)
# print(len(test_data))
# print(len(train_data))
# 构建词表, Vocabulary.add(word)
vocab = Vocabulary(min_freq=2)
train_data.apply(lambda x: [vocab.add(word) for word in x['words']])
vocab.build_vocab()
# index句子, Vocabulary.to_index(word)
train_data.apply(
lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words')
test_data.apply(
lambda x: [vocab.to_index(word) for word in x['words']],
new_field_name='words')
print(test_data[0])
model = CNNText(embed_num=len(vocab),
embed_dim=50,
num_classes=5,
padding=2,
dropout=0.1)
from fastNLP import Trainer
from copy import deepcopy
# 更改DataSet中对应field的名称，要以模型的forward等参数名一致
train_data.rename_field('words',
'word_seq') # input field 与 forward 参数一致
train_data.rename_field('label', 'label_seq')
test_data.rename_field('words', 'word_seq')
test_data.rename_field('label', 'label_seq')
# 实例化Trainer，传入模型和数据，进行训练
copy_model = deepcopy(model)
overfit_trainer = Trainer(train_data=test_data,
model=copy_model,
loss=CrossEntropyLoss(pred="output",
target="label_seq"),
metrics=AccuracyMetric(pred="predict",
target="label_seq"),
n_epochs=10,
batch_size=4,
dev_data=test_data,
save_path="./save")
overfit_trainer.train()
trainer = Trainer(train_data=train_data,
model=model,
loss=CrossEntropyLoss(pred="output",
target="label_seq"),
metrics=AccuracyMetric(pred="predict",
target="label_seq"),
n_epochs=10,
batch_size=4,
dev_data=test_data,
save_path="./save")
trainer.train()
print('Train finished!')
# 使用fastNLP的Tester测试脚本
tester = Tester(data=test_data,
model=model,
metrics=AccuracyMetric(pred="predict",
target="label_seq"),
batch_size=4)
acc = tester.test()
print(acc)