def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# build model
if params.encoder_only:
model = build_model(params, data['dico'])
else:
encoder, decoder = build_model(params, data['dico'])
# float16
if params.fp16:
assert torch.backends.cudnn.enabled
if params.encoder_only:
model = network_to_half(model)
else:
encoder = network_to_half(encoder)
decoder = network_to_half(decoder)
# distributed
if params.multi_gpu:
logger.info("Using nn.parallel.DistributedDataParallel ...")
if params.encoder_only:
model = apex.parallel.DistributedDataParallel(model,
delay_allreduce=True)
else:
encoder = apex.parallel.DistributedDataParallel(
encoder, delay_allreduce=True)
decoder = apex.parallel.DistributedDataParallel(
decoder, delay_allreduce=True)
# build trainer, reload potential checkpoints / build evaluator
if params.encoder_only:
trainer = SingleTrainer(model, data, params)
evaluator = SingleEvaluator(trainer, data, params)
else:
trainer = EncDecTrainer(encoder, decoder, data, params)
evaluator = EncDecEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
trainer.clm_step(lang1, lang2, params.lambda_clm)
# MLM steps (also includes TLM if lang2 is not None)
for lang2, lang2 in shuf_order(params.mlm_steps, params):
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder steps
for lang in shuf_order(params.ae_steps):
trainer.mt_step(lang, lang, params.lambda_ae)
# mass prediction steps
for lang in shuf_order(params.mass_steps):
trainer.mass_step(lang, params.lambda_mass)
# machine translation steps
for lang1, lang2 in shuf_order(params.mt_steps, params):
trainer.mt_step(lang1, lang2, params.lambda_mt)
# back-translation steps
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
# back-parallel steps
for lang1, lang2 in shuf_order(params.bmt_steps, params):
trainer.bmt_step(lang1, lang2, params.lambda_bmt)
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
print(data)
# build model
# if params.encoder_only:
model = build_model(params)
# build trainer, reload potential checkpoints / build evaluator
trainer = XTrainer(model, data, params)
evaluator = XEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
if params.is_understanding:
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# cross-modal caption steps
for lang1, lang2 in shuf_order(params.cross_modal_steps, params):
if params.is_mt:
trainer.mt_ic_step(lang1, lang2, params.lambda_ic)
else:
trainer.ic_step(lang1, lang2, params.lambda_ic)
for lang1, lang2 in shuf_order(params.cross_rel_steps, params):
trainer.rel_step(lang1, lang2, params.lambda_t2i,
params.lambda_i2t)
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
evaluate_results = []
import os
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
evaluate_results.append(json.dumps(scores))
with open(
os.path.join(params.dump_path,
"epoch_{0}.eval_log".format(trainer.epoch)),
'w') as writer:
for line in evaluate_results:
writer.write(line + '\n')
# end of epoch
trainer.save_best_model(scores)
if trainer.epoch % params.save_every_epoch == 0 and params.is_master:
trainer.save_model('model_pretrain_%i' % trainer.epoch)
trainer.save_periodic()
trainer.end_epoch(scores)
def main(params):
warnings.filterwarnings("ignore", category=UserWarning)
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# build model
if params.encoder_only:
model = build_model(params, data['dico'])
else:
encoder, decoder = build_model(params, data['dico'])
# build trainer, reload potential checkpoints / build evaluator
if params.encoder_only:
trainer = SingleTrainer(model, data, params)
evaluator = SingleEvaluator(trainer, data, params)
else:
trainer = EncDecTrainer(encoder, decoder, data, params)
evaluator = EncDecEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
trainer.clm_step(lang1, lang2, params.lambda_clm)
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder steps
for lang in shuf_order(params.ae_steps):
trainer.mt_step(lang, lang, params.lambda_ae)
# machine translation steps
for lang1, lang2 in shuf_order(params.mt_steps, params):
trainer.mt_step(lang1, lang2, params.lambda_mt)
# back-translation steps
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
if trainer.n_total_iter < 3:
trainer.get_gpu_statistics()
trainer.iter()
if trainer.n_total_iter % 500 == 0:
evaluator.run_all_evals(trainer, valid_only=True)
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
meta_params = copy.deepcopy(params).meta_params
params.meta_params = "..." # to long to be log
logger = initialize_exp(params)
params.meta_params = meta_params
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
print(params.meta_params.keys())
print(data.keys())
# todo : good params.n_words (We take the one from the first task have this parameter for the moment.)
"""
But we think that if all the task data are based on the same vocabulary, all these parameters will be the same,
and therefore no problem if we choose one at random.
"""
p = params.meta_params[data['key']]
# build model
if params.encoder_only:
model = build_model(params=p, dico=data['dico'])
else:
encoder, decoder = build_model(params=p, dico=data['dico'])
# todo : good pad_index and eos_index and ... (I'll take the one from the first task for the moment.)
"""
But we think that if all the task data are based on the same vocabulary, all these parameters will be the same,
and therefore no problem if we choose one at random.
"""
params.n_words = p.n_words
params.bos_index = p.bos_index
params.eos_index = p.eos_index
params.pad_index = p.pad_index
params.unk_index = p.unk_index
params.mask_index = p.mask_index
# build trainer, reload potential checkpoints / build evaluator
if params.encoder_only:
trainer = SingleTrainer(model, data, params)
evaluator = SingleEvaluator(trainer, data, params)
else:
trainer = EncDecTrainer(encoder, decoder, data, params)
evaluator = EncDecEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
if not params.meta_learning:
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
else:
for lgs in params.meta_params.keys():
logger.info("============ task : %s " % lgs)
for k, v in scores[lgs].items():
if k != "epoch":
logger.info("%s -> %.6f" % (k, v))
logger.info("============ all")
for k, v in scores.items():
if not (k in (list(params.meta_params.keys()) + ['epoch'])):
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
if not params.meta_learning:
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
trainer.clm_step(lang1, lang2, params.lambda_clm)
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder steps
for lang in shuf_order(params.ae_steps):
trainer.mt_step(lang, lang, params.lambda_ae)
# machine translation steps
for lang1, lang2 in shuf_order(params.mt_steps, params):
trainer.mt_step(lang1, lang2, params.lambda_mt)
# back-translation steps
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
trainer.iter()
else:
# our
trainer.n_sentences = {}
"""
Here we build language lists for each of our meta-taks. Indeed, for two language lists l1 and l2,
the objective will be done with l1[i] and l2[i] respectively, this for each index i of the two lists.
"""
lang1_dic, lang2_dic, lang3_dic = {}, {}, {}
"""
In the case of meta-learning, we have a (meta-)data dictionary for each (meta-)task,
so the keys are the languages conserved by the task.
"""
data_keys_dic = {}
# equivalent to "for task in list of task" in the original algorithm, except here we prepare all the tasks beforehand.
for lgs in params.meta_params.keys():
trainer.n_sentences[lgs] = 0
# CLM
try:
lang1_dic['clm_step']
except KeyError:
lang1_dic['clm_step'], lang2_dic[
'clm_step'], data_keys_dic['clm_step'] = [], [], []
for lang1, lang2 in shuf_order(
params.meta_params[lgs].clm_steps, params):
lang1_dic['clm_step'].append(lang1)
lang2_dic['clm_step'].append(lang2)
data_keys_dic['clm_step'].append(lgs)
# MLM
try:
lang1_dic['mlm_step']
except KeyError:
lang1_dic['mlm_step'], lang2_dic[
'mlm_step'], data_keys_dic['mlm_step'] = [], [], []
for lang1, lang2 in shuf_order(
params.meta_params[lgs].mlm_steps, params):
lang1_dic['mlm_step'].append(lang1)
lang2_dic['mlm_step'].append(lang2)
data_keys_dic['mlm_step'].append(lgs)
# parallel classification
try:
lang1_dic['pc_step']
except KeyError:
lang1_dic['pc_step'], lang2_dic['pc_step'], data_keys_dic[
'pc_step'] = [], [], []
for lang1, lang2 in shuf_order(
params.meta_params[lgs].pc_steps, params):
lang1_dic['pc_step'].append(lang1)
lang2_dic['pc_step'].append(lang2)
data_keys_dic['pc_step'].append(lgs)
# denoising auto-encoder
try:
lang1_dic['ae_step']
except KeyError:
lang1_dic['ae_step'], data_keys_dic['ae_step'] = [], []
for lang1 in shuf_order(params.meta_params[lgs].ae_steps):
lang1_dic['ae_step'].append(lang1)
data_keys_dic['ae_step'].append(lgs)
# machine translation
try:
lang1_dic['mt_step']
except KeyError:
lang1_dic['mt_step'], lang2_dic['mt_step'], data_keys_dic[
'mt_step'] = [], [], []
for lang1, lang2 in shuf_order(
params.meta_params[lgs].mt_steps, params):
lang1_dic['mt_step'].append(lang1)
lang2_dic['mt_step'].append(lang2)
data_keys_dic['mt_step'].append(lgs)
# back-translation
try:
lang1_dic['bt_step']
except KeyError:
lang1_dic['bt_step'], lang2_dic['bt_step'], lang3_dic[
'bt_step'], data_keys_dic['bt_step'] = [], [], [], []
for lang1, lang2, lang3 in shuf_order(
params.meta_params[lgs].bt_steps):
lang1_dic['bt_step'].append(lang1)
lang2_dic['bt_step'].append(lang2)
lang3_dic['bt_step'].append(lang3)
data_keys_dic['bt_step'].append(lgs)
flag = True
# equivalent to "while not done do" in the original algorithm
while flag:
# CLM steps
#print("clm_step", flag)
a = trainer.clm_step(lang1_dic['clm_step'],
lang2_dic['clm_step'], params.lambda_clm,
data_keys_dic['clm_step'])
#print("mlm_step", flag)
# MLM steps (also includes TLM if lang2 is not None)
b = trainer.mlm_step(lang1_dic['mlm_step'],
lang2_dic['mlm_step'], params.lambda_mlm,
data_keys_dic['mlm_step'])
# parallel classification steps
c = trainer.pc_step(lang1_dic['pc_step'], lang2_dic['pc_step'],
params.lambda_pc, data_keys_dic['pc_step'])
if isinstance(trainer, EncDecTrainer):
# denoising auto-encoder steps
d = trainer.mt_step(lang1_dic['ae_step'],
lang1_dic['ae_step'], params.lambda_ae,
data_keys_dic['ae_step'])
# machine translation steps
e = trainer.mt_step(lang1_dic['mt_step'],
lang2_dic['mt_step'], params.lambda_mt,
data_keys_dic['mt_step'])
# back-translation steps
f = trainer.bt_step(lang1_dic['bt_step'],
lang2_dic['bt_step'],
lang3_dic['bt_step'], params.lambda_bt,
data_keys_dic['bt_step'])
# do things better
if (not a) and (not b) and (not c) and (not d) and (
not e) and (not f):
flag = False # End of epoch
else:
flag = True
else:
# do things better
if (not a) and (not b) and (not c):
flag = False # End of epoch
else:
flag = True
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
if not params.meta_learning:
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
else:
for lgs in params.meta_params.keys():
logger.info("============ task : %s " % lgs)
for k, v in scores[lgs].items():
if k != "epoch":
logger.info("%s -> %.6f" % (k, v))
logger.info("============ all")
for k, v in scores.items():
if not (k in (list(params.meta_params.keys()) + ['epoch'])):
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
# our
logger.info("============ garbage collector collecting %d ..." %
gc.collect())
def main(params):
if params.adv:
params.use_lang_emb = False
print("Language embeddings are not used...\n \n \n \n")
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# build model
# reload-model options are in here
if params.encoder_only:
model = build_model(params, data['dico'])
else:
encoder, decoder = build_model(params, data['dico'])
# build trainer, reload potential checkpoints / build evaluator
if params.encoder_only:
trainer = SingleTrainer(model, data, params)
evaluator = SingleEvaluator(trainer, data, params)
else:
trainer = EncDecTrainer(encoder, decoder, data, params)
evaluator = EncDecEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for epoch in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" % trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
trainer.clm_step(lang1, lang2, params.lambda_clm)
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder
for lang in shuf_order(params.ae_steps):
trainer.mt_step(lang, lang, params.lambda_ae)
for lang1, lang2 in shuf_order(params.mt_steps, params):
trainer.mt_step(lang1, lang2, params.lambda_mt)
# back-translation
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
trainer.iter()
logger.info("============ End of epoch %i ============" % trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# load checkpoint
if params.model_path != "":
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
encoder = TransformerModel(model_params,
dico,
is_encoder=True,
with_output=True).cuda().eval()
decoder = TransformerModel(model_params,
dico,
is_encoder=False,
with_output=True).cuda().eval()
encoder = TransformerModel(model_params,
dico,
is_encoder=True,
with_output=True).cuda().eval()
decoder = TransformerModel(model_params,
dico,
is_encoder=False,
with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder.load_state_dict(reloaded['decoder'])
logger.info("Supported languages: %s" %
", ".join(model_params.lang2id.keys()))
else:
# build model
if params.encoder_only:
model = build_model(params, data['dico'])
else:
encoder, decoder = build_model(params, data['dico'])
# build trainer, reload potential checkpoints / build evaluator
if params.encoder_only:
trainer = SingleTrainer(model, data, params)
evaluator = SingleEvaluator(trainer, data, params)
else:
trainer = EncDecTrainer(encoder, decoder, data, params)
evaluator = EncDecEvaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
trainer.clm_step(lang1, lang2, params.lambda_clm)
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
trainer.mlm_step(lang1, lang2, params.lambda_mlm)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder steps
for lang in shuf_order(params.ae_steps):
trainer.mt_step(lang, lang, params.lambda_ae)
# machine translation steps
for lang1, lang2 in shuf_order(params.mt_steps, params):
trainer.mt_step(lang1, lang2, params.lambda_mt)
# back-translation steps
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# build model
model = build_model(params, data['dico'])
# build trainer, reload potential checkpoints / build evaluator
trainer = Trainer(model, data, params)
evaluator = Evaluator(trainer, data, params)
# evaluation
if params.eval_only:
scores = evaluator.run_all_evals(trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for _ in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
trainer.epoch)
trainer.n_sentences = 0
while trainer.n_sentences < trainer.epoch_size:
# MLM steps
trainer.mlm_step(params.lambda_mlm)
trainer.iter()
logger.info("============ End of epoch %i ============" %
trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
trainer.save_best_model(scores)
trainer.save_periodic()
trainer.end_epoch(scores)
def main(params):
# initialize the multi-GPU / multi-node training
init_distributed_mode(params)
# initialize the experiment
logger = initialize_exp(params)
# initialize SLURM signal handler for time limit / pre-emption
init_signal_handler()
# load data
data = load_data(params)
# build the big model
if params.encoder_only:
big_model = build_model(params, data['dico'], cut=False)
else:
# 修改处1
big_encoder, big_decoder = build_model(params, data['dico'], cut=False)
# if we cut some layers, must build a small model
if params.cut_layer:
if params.encoder_only:
small_model = build_model(params, data['dico'], cut=True)
else:
# 修改处1
small_encoder, small_decoder = build_model(params,
data['dico'],
cut=True)
# build the big trainer, reload potential checkpoints
# the big trainer is used to train, so need't a evaluator for it
if params.encoder_only:
big_trainer = SingleTrainer(big_model, data, params)
else:
big_trainer = EncDecTrainer(big_encoder, big_decoder, data, params)
params.lambda_mlm = "1"
params.lambda_clm = "1"
params.lambda_pc = "1"
params.lambda_ae = "1"
params.lambda_mt = "1"
params.lambda_bt = "1"
# build the small model, and use it for evaluator
if params.encoder_only:
small_trainer = small_SingleTrainer(small_model, data, params)
evaluator = SingleEvaluator(small_trainer, data, params)
else:
small_trainer = small_EncDecTrainer(small_encoder, small_decoder, data,
params)
evaluator = EncDecEvaluator(small_trainer, data, params)
# evaluation only for the small trainer
if params.eval_only:
scores = evaluator.run_all_evals(small_trainer)
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
logger.info("__log__:%s" % json.dumps(scores))
exit()
# set sampling probabilities for training
set_sampling_probs(data, params)
# language model training
for count in range(params.max_epoch):
logger.info("============ Starting epoch %i ... ============" %
small_trainer.epoch)
small_trainer.n_sentences = 0
while small_trainer.n_sentences < small_trainer.epoch_size:
# CLM steps
for lang1, lang2 in shuf_order(params.clm_steps, params):
small_trainer.clm_step(lang1, lang2, params.lambda_clm,
big_trainer)
# MLM steps (also includes TLM if lang2 is not None)
for lang1, lang2 in shuf_order(params.mlm_steps, params):
small_trainer.mlm_step(lang1, lang2, params.lambda_mlm,
big_trainer)
# parallel classification steps
for lang1, lang2 in shuf_order(params.pc_steps, params):
small_trainer.pc_step(lang1, lang2, params.lambda_pc)
# denoising auto-encoder steps
for lang in shuf_order(params.ae_steps):
small_trainer.mt_step(lang, lang, params.lambda_ae,
big_trainer)
# machine translation steps
for lang1, lang2 in shuf_order(params.mt_steps, params):
small_trainer.mt_step(lang1, lang2, params.lambda_mt,
big_trainer)
# back-translation steps
for lang1, lang2, lang3 in shuf_order(params.bt_steps):
small_trainer.bt_step(lang1, lang2, lang3, params.lambda_bt)
small_trainer.iter()
logger.info("============ End of epoch %i ============" %
small_trainer.epoch)
# evaluate perplexity
scores = evaluator.run_all_evals(small_trainer)
# print / JSON log
for k, v in scores.items():
logger.info("%s -> %.6f" % (k, v))
if params.is_master:
logger.info("__log__:%s" % json.dumps(scores))
# end of epoch
small_trainer.save_best_model(scores)
small_trainer.save_periodic()
small_trainer.end_epoch(scores)
