def record_hyper_params(hyper_dict: dict):
for k, v in hyper_dict.items():
if k not in ('model_dir', 'record_dir', 'data_path'):
v = v if v is not None else '-'
fitlog.add_hyper(value=str(v), name=str(k))
if 'dilate' not in hyper_dict.keys():
fitlog.add_hyper(value='-', name='dilate')
fitlog.add_other(value=platform.system(), name='platform')
def on_exception(self, exception):
fitlog.finish(status=1)
if self._log_exception:
fitlog.add_other(repr(exception), name='except_info')
if 'p' in os.environ:
os.environ['CUDA_VISIBLE_DEVICES']=os.environ['p']
import torch
from torch import optim
from fastNLP import Trainer, CrossEntropyLoss
from fastNLP import BucketSampler, cache_results, WarmupCallback, GradientClipCallback, FitlogCallback
from mono.data.pipe import ENBertPipe
from mono.model.bert import ENBertReverseDict
import fitlog
from mono.model.metrics import MonoMetric
from joint.data.utils import clip_max_length
# fitlog.debug()
fitlog.set_log_dir('en_logs')
fitlog.add_hyper_in_file(__file__)
fitlog.add_other('uncased', name='note')
paths = '../data/en'
#######hyper
model_name = 'bert'
max_word_len = 5
lr = 2e-5
batch_size = 64
n_epochs = 10
#######hyper
pre_name = 'bert-base-uncased'
# 在transformers中的名字叫做bert-base-cased
@cache_results('caches/en_{}_{}.pkl'.format(pre_name.split('/')[-1], max_word_len), _refresh=False)
def get_data():
def fit(self):
last_miou = .0 # record the best validation mIoU
loss_step = 0 # step count
for epoch in range(self.conf.epochs):
train_loss = .0
start = time.time()
for i, (data, target) in enumerate(self.train_iter):
gpu_datas = split_and_load(data, ctx_list=self.ctx)
gpu_targets = split_and_load(target, ctx_list=self.ctx)
with autograd.record():
loss_gpu = [
self.criterion(*self.net(gpu_data), gpu_target)
for gpu_data, gpu_target in zip(
gpu_datas, gpu_targets)
]
for loss in loss_gpu:
autograd.backward(loss)
self.trainer.step(self.conf.bs_train)
nd.waitall()
loss_temp = .0
for losses in loss_gpu:
loss_temp += losses.sum().asscalar()
train_loss += (loss_temp / self.conf.bs_train)
# log every n batch
# add loss to draw curve, train_loss <class numpy.float64>
interval = 5 if loss_step < 5000 else 50
if (i % interval == 0) or (i + 1 == len(self.train_iter)):
fitlog.add_loss(name='loss',
value=round(train_loss / (i + 1), 5),
step=loss_step)
loss_step += 1
self.logger.info(
"Epoch %d, batch %d, training loss %.5f." %
(epoch, i, train_loss / (i + 1)))
# log each epoch
self.logger.info(
">>>>>> Epoch %d complete, time cost: %.1f sec. <<<<<<" %
(epoch, time.time() - start))
# validation each epoch
if self.val:
pixel_acc, mean_iou = self._validation()
self.logger.info(
"Epoch %d validation, PixelAccuracy: %.4f, mIoU: %.4f." %
(epoch, pixel_acc, mean_iou))
fitlog.add_metric(value=mean_iou, step=epoch, name='mIoU')
fitlog.add_metric(value=pixel_acc, step=epoch, name='PA')
if mean_iou > last_miou:
f_name = self._save_model(tag='best')
self.logger.info(
"Epoch %d mIoU: %.4f > %.4f(previous), save model: %s"
% (epoch, mean_iou, last_miou, f_name))
last_miou = mean_iou
# save the final-epoch params
f_name = self._save_model(tag='last')
self.logger.info(">>>>>> Training complete, save model: %s. <<<<<<" %
f_name)
# record
fitlog.add_best_metric(value=round(last_miou, 4), name='mIoU')
fitlog.add_other(value=self.id, name='record_id')
fitlog.add_other(value=self.num_train, name='train')
fitlog.add_other(value=self.num_val, name='val')
def log_config(config, task_name):
fitlog.add_other(task_name, name="model_name")
# irrelavant
fitlog.add_other(config.class_num, name="class_num")
fitlog.add_other(str(config.kernel_sizes), name="kernel_sizes")
fitlog.add_other(config.kernel_num, name="kernel_num")
fitlog.add_other(config.in_channels, name="in_channels")
fitlog.add_other(config.dropout, name="dropout")
fitlog.add_other(config.static, name="static")
# irrelavant above
fitlog.add_other(config.sentence_length, name="sentence_length")
fitlog.add_other(config.num_layers, name="num_layers") # irrelavant
fitlog.add_other(config.hidden_dim, name="hidden_dim") # irrelavant
fitlog.add_other(config.lr, name="lr")
fitlog.add_other(config.weight_decay, name="weight_decay") # irrelavant
fitlog.add_other(config.patience, name="patience")
fitlog.add_other(config.epoch, name="epoch")
fitlog.add_other(config.batch_size, name="batch_size")
fitlog.add_other(str(config.device), name="device")
fitlog.add_other(config.warmup_proportion, name="warmup_proportion")
fitlog.add_other(config.local_rank, name="local_rank")
fitlog.add_other(config.update_every, name="update_every")
from fastNLP import BucketSampler, cache_results, WarmupCallback, GradientClipCallback
from mix.data.pipe import MixUnalignBertPipe
from mix.data.loader import MixUnAlignLoader
# from V1.model.bert import ENBertReverseDict
from mix.model.bert import JointBertReverseDict
from fastNLP import DataSet, Tester
import fitlog
from mix.model.loss import MixLoss
from joint.model.metrics import JointMetric, SummaryMetric
from joint.data.utils import clip_max_length
from mix.model.metrics import SaveMetric
from joint.model.callback import FitlogCallback
# fitlog.debug()
fitlog.set_log_dir('logs')
fitlog.add_hyper_in_file(__file__)
fitlog.add_other('no lg loss', name='note')
paths = '../data/mix'
#######hyper
model_name = 'bert'
max_word_len = 5
lr = 5e-5
lg_lambda = 0.0
batch_size = 80
n_epochs = 20
#######hyper
pre_name = 'bert-base-multilingual-cased'
# transformers中bert-base-multilingual-cased
]
callbacks.append(FitlogCallback(data=data, verbose=1))
train_data = data_bundle.get_dataset('train')
train_data.add_seq_len('input')
sampler = BucketSampler()
clip_max_length(train_data, data_bundle)
trainer = Trainer(train_data=train_data,
model=model,
optimizer=optimizer,
loss=CrossEntropyLoss(),
batch_size=batch_size,
sampler=sampler,
drop_last=False,
update_every=1,
num_workers=1,
n_epochs=n_epochs,
print_every=5,
dev_data=data_bundle.get_dataset('dev'),
metrics=MonoMetric(),
metric_key='t1',
validate_every=-1,
save_path='save_models/',
use_tqdm=True,
device=None,
callbacks=callbacks,
check_code_level=0)
trainer.train(load_best_model=False)
fitlog.add_other(trainer.start_time, name='start_time')
def log_config(config, task_name):
fitlog.add_other(task_name, name="model_name")
fitlog.add_other(str(config.device), name="device")
fitlog.add_other(config.char_embed_dim, name="char_embed_dim")
fitlog.add_other(config.word_embed_dim, name="word_embed_dim")
fitlog.add_other(config.pos_embed_dim, name="pos_embed_dim")
fitlog.add_other(config.sentence_length, name="sentence_length")
fitlog.add_other(config.hidden_dim, name="hidden_dim")
fitlog.add_other(config.dropout, name="dropout")
fitlog.add_other(config.encoding_type, name="encoding_type")
fitlog.add_other(config.epoch, name="epoch")
fitlog.add_other(config.batch_size, name="batch_size")
fitlog.add_other(config.lr, name="lr")
fitlog.add_other(config.weight_decay, name="weight_decay")
fitlog.add_other(config.patience, name="patience")
fitlog.add_other(config.num_layers, name="num_layers")
fitlog.add_other(config.inner_size, name="inner_size")
fitlog.add_other(config.key_size, name="key_size")
fitlog.add_other(config.value_size, name="value_size")
fitlog.add_other(config.num_head, name="num_head")
metrics=[AccuracyMetric(),
ClassifyFPreRecMetric(f_type="macro")],
metric_key=None,
validate_every=-1,
save_path=None,
use_tqdm=False,
device=0,
callbacks=callbacks,
check_code_level=0,
test_sampler=SortedSampler(),
test_use_tqdm=False,
)
trainer.train(load_best_model=False)
fitlog.add_other(trainer.start_time, name="start_time")
os.makedirs("./save_models", exist_ok=True)
folder = "./save_models/{}-{}-{}".format(model_type, args.dataset, "FT")
count = 0
for fn in os.listdir("save_models"):
if fn.startswith(folder.split("/")[-1]):
count += 1
folder = folder + str(count)
fitlog.add_other(count, name="count")
if args.save_embed and not os.path.exists(folder):
if not isinstance(embed, XLMRobertaModel):
embed.save(folder)
else:
os.makedirs(folder, exist_ok=True)
os.makedirs("{}/{}".format(folder, model_type), exist_ok=True)
embed.save_pretrained("{}/{}".format(folder, model_type))
import torch
from torch import optim
from fastNLP import Trainer, CrossEntropyLoss
from fastNLP import BucketSampler, cache_results, WarmupCallback, GradientClipCallback
from joint.data.pipe import BiAlignedBertPipe
from joint.data.loader import BiAlignLoader
# from V1.model.bert import ENBertReverseDict
from joint.model.bert import BiBertReverseDict
import fitlog
from joint.model.metrics import BiMetric
from joint.model.callback import FitlogCallback
from joint.data.utils import clip_max_length
# fitlog.debug()
fitlog.set_log_dir('logs')
fitlog.add_hyper_in_file(__file__)
fitlog.add_other(name='note', value='use embedding')
paths = '../data/mix'
#######hyper
pair = 'es_en'
model_name = 'bert'
max_word_len = 5
lr = 1e-4
batch_size = 80
n_epochs = 20
#######hyper
pre_name = 'bert-base-multilingual-cased'
# transformers中bert-base-multilingual-cased
def log_config(config, task_name):
fitlog.add_other(task_name, name="model_name")
fitlog.add_other(config.class_num, name="class_num")
fitlog.add_other(str(config.kernel_sizes), name="kernel_sizes")
fitlog.add_other(config.kernel_num, name="kernel_num")
fitlog.add_other(config.in_channels, name="in_channels")
fitlog.add_other(config.dropout, name="dropout")
fitlog.add_other(config.static, name="static")
fitlog.add_other(config.sentence_length, name="sentence_length")
fitlog.add_other(config.num_layers, name="num_layers")
fitlog.add_other(config.hidden_dim, name="hidden_dim")
fitlog.add_other(config.lr, name="lr")
fitlog.add_other(config.weight_decay, name="weight_decay")
fitlog.add_other(config.patience, name="patience")
fitlog.add_other(config.epoch, name="epoch")
fitlog.add_other(config.batch_size, name="batch_size")
fitlog.add_other(str(config.device), name="device")
from torch import optim
from fastNLP import Trainer
from fastNLP import BucketSampler, cache_results, WarmupCallback, GradientClipCallback
from joint.data.pipe import JointAlignBertPipe
from joint.data.loader import JointAlignLoader
from joint.model.bert import JointBertReverseDict
import fitlog
from joint.model.metrics import JointMetric, SummaryMetric
from fastNLP import DataSet, Tester
from joint.model.callback import FitlogCallback
from mix.model.loss import MixLoss
from joint.data.utils import clip_max_length
# fitlog.debug()
fitlog.set_log_dir('logs')
fitlog.add_hyper_in_file(__file__)
fitlog.add_other(name='note', value='lm head')
paths = '../data/mix'
#######hyper
model_name = 'bert'
max_word_len = 5
batch_size = 80
n_epochs = 20
lg_lambda = 0.0
lr = 5e-5
#######hyper
pre_name = 'bert-base-multilingual-cased'
# transformers中xlm-roberta-base
@cache_results('caches/joint_{}_{}.pkl'.format(
