def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
parser.add('--data', '-data', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data/demo',
help='Path prefix to the ".train.pt" and '
'".valid.pt" file path from preprocess.py')
parser.add('--save_model', '-save_model', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data',
help="Model filename (the model will be saved as "
"<save_model>_N.pt where N is the number "
"of steps")
parser.add('--save_checkpoint_steps', '-save_checkpoint_steps',
type=int, default=500,
help="""Save a checkpoint every X steps""")
parser.add('--train_from', '-train_from',
# default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt',
default='',
type=str,
help="If training from a checkpoint then this is the "
"path to the pretrained model's state_dict.")
# default = 100000,
parser.add('--train_steps', '-train_steps', type=int, default=100000,
help='训练多少步')
return parser
def build_translator(opt, report_score=True, logger=None, out_file=None):
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
dummy_parser = configargparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
load_test_model = onmt.decoders.ensemble.load_test_model \
if len(opt.models) > 1 else onmt.model_builder.load_test_model
fields, model, model_opt = load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer.from_opt(opt)
translator = Translator(
model,
fields,
opt,
model_opt,
global_scorer=scorer,
out_file=out_file,
report_score=report_score,
logger=logger
)
return translator
def build_translator(opt, report_score=True, logger=None, out_file=None, use_output=True):
if out_file is None and use_output:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
if opt.gpu > -1:
torch.cuda.set_device(opt.gpu)
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
opt.beta,
opt.coverage_penalty,
opt.length_penalty)
kwargs = {k: getattr(opt, k)
for k in ["beam_size", "n_best", "max_length", "min_length",
"stepwise_penalty", "block_ngram_repeat",
"ignore_when_blocking", "dump_beam", "dump_layers",
"report_bleu", "data_type", "replace_unk", "gpu",
"verbose"]}
translator = Translator(model, fields, global_scorer=scorer,
out_file=out_file, report_score=report_score,
copy_attn=model_opt.copy_attn, logger=logger,
**kwargs)
return translator
def build_translator(opt):
dummy_parser = configargparse.ArgumentParser(description='translate.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
fields, model = load_test_model(opt, dummy_opt.__dict__)
translator = Translator(model, fields, opt)
return translator
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's usefull in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
else:
checkpoint = None
model_opt = opt
# Load fields generated from preprocess phase.
fields = load_fields(opt, checkpoint)
# Build model.
model = build_model(model_opt, opt, fields, checkpoint)
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
# Build optimizer.
optim = build_optim(model, opt, checkpoint)
# Build model saver
model_saver = build_model_saver(model_opt, opt, model, fields, optim)
trainer = build_trainer(opt, device_id, model, fields,
optim, model_saver=model_saver)
def train_iter_fct():
return build_dataset_iter(
load_dataset("train", opt), fields, opt)
def valid_iter_fct():
return build_dataset_iter(
load_dataset("valid", opt), fields, opt, is_train=False)
# Do training.
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
opt.valid_steps)
if opt.tensorboard:
trainer.report_manager.tensorboard_writer.close()
def build_translator(opt, report_score=True, logger=None, out_file=None):
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
if opt.gpu > -1:
torch.cuda.set_device(opt.gpu)
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
if len(opt.models) > 1:
# use ensemble decoding if more than one model is specified
fields, model, model_opt = \
onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
else:
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(opt)
translator = CodeTranslator(model,
fields,
opt,
model_opt,
global_scorer=scorer,
out_file=out_file,
report_score=report_score,
logger=logger)
translator.src_path = opt.src
return translator
def build_translator(opt, report_score=True, logger=None, out_file=None):
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
if len(opt.models) > 1:
# use ensemble decoding if more than one model is specified
fields, model, model_opt = \
onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
else:
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
opt.beta,
opt.coverage_penalty,
opt.length_penalty)
kwargs = {k: getattr(opt, k)
for k in ["beam_size", "n_best", "max_length", "min_length",
"stepwise_penalty", "block_ngram_repeat",
"ignore_when_blocking", "dump_beam", "report_bleu",
"data_type", "replace_unk", "gpu", "verbose", "fast",
"sample_rate", "window_size", "window_stride",
"window", "image_channel_size"]}
translator = Translator(model, fields, global_scorer=scorer,
out_file=out_file, report_score=report_score,
copy_attn=model_opt.copy_attn, logger=logger,
**kwargs)
return translator
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='build_copy_transformer.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def main():
parser = ArgumentParser()
opts.config_opts(parser)
opts.model_opts(parser)
opts.global_opts(parser)
opt = parser.parse_args()
with open(os.path.join(dir_path, 'opt_data'), 'wb') as f:
pickle.dump(opt, f)
def _get_parser():
parser = ArgumentParser(description='RL_train.py')
opts.config_opts(parser)
# yida RL
opts.model_opts(parser)
# opts.train_opts(parser)
opts.translate_opts(parser)
return parser
def build_translator(opt,
report_score=True,
out_file=None,
logger=None,
fields=None,
model=None,
model_opt=None):
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
# model should already be on device, is being used in part fo the training loop now
# print(opt)
# quit()
# if opt.gpu > -1:
# torch.cuda.set_device(opt.gpu)
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
# if len(opt.models) > 1:
# # use ensemble decoding if more than one model is specified
# fields, model, model_opt = \
# onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
# else:
if fields is None and model is None and model_opt is None:
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(
0, #opt.alpha,
0, #opt.beta,
None, #opt.coverage_penalty,
None) #opt.length_penalty)
kwargs = {
k: getattr(opt, k)
for k in [
"beam_size", "n_best", "max_length", "min_length",
"stepwise_penalty", "block_ngram_repeat", "ignore_when_blocking",
"dump_beam", "report_bleu", "data_type", "replace_unk", "verbose",
"fast"
]
} # TODO gpu
if opt.gpu_ranks:
kwargs['gpu'] = True
translator = Translator(model,
fields,
global_scorer=scorer,
out_file=out_file,
report_score=report_score,
copy_attn=model_opt.copy_attn,
logger=logger,
**kwargs)
return translator
def build_translator(opt):
out_file = open(opt.output, 'w+', encoding='utf-8')
dummy_parser = configargparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
load_test_model = onmt.decoders.ensemble.load_test_model \
if len(opt.models) > 1 else onmt.model_builder.load_test_model
fields, model, _ = load_test_model(opt, dummy_opt.__dict__)
return Translator(model, fields, opt, out_file=out_file)
def build_translator(opt,model_path): dummy_parser = configargparse.ArgumentParser(description='translate.py') opts.model_opts(dummy_parser) dummy_opt = dummy_parser.parse_known_args([])[0] #build the model and get the checkpoint and field path=[] path.append(model_path) opt.models=path fields, model = nmt_model.load_test_model(opt, dummy_opt.__dict__) translator = Translator(model, fields, opt) return translator
def build_encoder_pass(opt, out_file=None):
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
dummy_parser = configargparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
load_test_model = onmt.decoders.ensemble.load_test_model \
if len(opt.models) > 1 else onmt.model_builder.load_test_model
fields, model, model_opt = load_test_model(opt, dummy_opt.__dict__)
encoder_pass = EncoderPass(model, fields, opt, out_file)
return encoder_pass
def parse_args():
parser = configargparse.ArgumentParser(
description='train.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.general_opts(parser)
opts.config_opts(parser)
opts.add_md_help_argument(parser)
opts.model_opts(parser)
opts.train_opts(parser)
opt = parser.parse_args()
return opt
def _get_parser():
parser = ArgumentParser(description='train.py')
parser.add_argument('--teacher_model_path',
action='store',
dest='teacher_model_path',
help='the path direct to the teacher model path')
parser.add_argument("--word_sampling",
action="store",
default=False,
help="optional arg")
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
'''extended opts for pretrained language models'''
group = parser.add_argument_group("extended opts")
group.add('--pretrained_encoder',
'-pretrained_encoder',
default="bert",
type=str,
choices=["bert", "roberta", "xlnet"],
help="choose a pretrained language model as encoder")
return parser
def main():
## from translate.py
parser = argparse.ArgumentParser(
description='translate.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
onmt.opts.add_md_help_argument(parser)
onmt.opts.translate_opts(parser)
onmt.opts.model_opts(parser)
logger.info("opts done")
print("opts done")
opt = parser.parse_args()
logger.info("opt.gpu " + str(opt.gpu))
print("opt.gpu " + str(opt.gpu))
if opt.gpu > -1:
torch.cuda.device(opt.gpu)
n_gpu = 1
else:
n_gpu = 0
# from train.py
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
logger.info("bulding loss")
print("building loss")
valid_loss = onmt.utils.loss.build_loss_compute(
model, fields["tgt"].vocab, opt, train=False)
logger.info("Inference starts")
print("Inference starts")
infer = Inference(model, valid_loss, fields, trunc_size=0,
shard_size=32, data_type='text', norm_method="sents",
n_gpu=n_gpu, gpu_rank=1, gpu_verbose_level=0, report_manager=None)
logger.info("translation")
print("translation")
infer.translate(src_path=opt.src,
tgt_path=opt.tgt,
src_dir=opt.src_dir,
batch_size=opt.batch_size,
ans_path=opt.ans)
def build_translator(opt):
dummy_parser = argparse.ArgumentParser(description="train.py")
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
if len(opt.models) > 1:
# use ensemble decoding if more than one model is specified
fields, model, model_opt = onmt.decoders.ensemble.load_test_model(
opt, dummy_opt.__dict__)
else:
fields, model, model_opt = onmt.model_builder.load_test_model(
opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(opt.alpha, opt.beta,
opt.coverage_penalty,
opt.length_penalty)
kwargs = {
k: getattr(opt, k)
for k in [
"beam_size",
"n_best",
"max_length",
"min_length",
"stepwise_penalty",
"block_ngram_repeat",
"ignore_when_blocking",
"data_type",
"replace_unk",
"gpu",
"fast",
"sample_rate",
"window_size",
"window_stride",
"window",
"image_channel_size",
"mask_from",
]
}
translator = Translator(model,
fields,
global_scorer=scorer,
copy_attn=model_opt.copy_attn,
**kwargs)
return translator
'opt': self.model_opt,
'optim': self.optim,
}
logger.info("Saving checkpoint %s_step_%d.pt" % (self.base_path, step))
checkpoint_path = '%s_step_%d.pt' % (self.base_path, step)
torch.save(checkpoint, checkpoint_path)
return checkpoint, checkpoint_path
def _rm_checkpoint(self, name):
"""
Remove a checkpoint
Args:
name(str): name that indentifies the checkpoint
(it may be a filepath)
"""
os.remove(name)
if __name__ == "__main__":
parser = configargparse.ArgumentParser(
description='train.py',
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.model_opts(parser)
opts.train_opts(parser)
opt = parser.parse_args()
main(opt)
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
out_file = None
best_test_score, best_ckpt = -10000, None
dummy_parser = argparse.ArgumentParser(description='meta_dev_zs.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
for i in range(57, opt.meta_iterations * opt.inner_iterations + 57, 57 * 5):
ckpt_path = '{}_epoch_{}.pt'.format(opt.save_model, i)
logger.info('Loading checkpoint from %s' % ckpt_path)
if out_file is None:
out_file = codecs.open(opt.output, 'w+', 'utf-8')
fields, model, model_opt = onmt.model_builder.load_test_model(opt, dummy_opt.__dict__, model_path=ckpt_path)
scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
opt.beta,
opt.coverage_penalty,
opt.length_penalty)
kwargs = {k: getattr(opt, k)
for k in ["beam_size", "n_best", "max_length", "min_length",
"stepwise_penalty", "block_ngram_repeat",
"ignore_when_blocking", "dump_beam", "report_bleu",
"replace_unk", "gpu", "verbose", "fast", "mask_from"]}
fields['graph'] = torchtext.data.Field(sequential = False)
translator = Translator(model, fields, global_scorer=scorer,
out_file=out_file, report_score=False,
copy_attn=model_opt.copy_attn, logger=logger,
log_probs_out_file=None,
**kwargs)
# make translation and save result
all_scores, all_predictions = translator.translate(
src_path='processed_data/meta-dev/' + opt.meta_dev_task + '/src-test.txt',
tgt_path=None,
src_dir=None,
batch_size=opt.translate_batch_size,
attn_debug=False)
# dump predictions
f = open('experiments/meta_dev/' + opt.meta_dev_task + '/test_predictions.csv', 'w', encoding='utf-8')
f.write('smiles,property\n')
for n_best_mols in all_predictions:
for mol in n_best_mols:
f.write(mol.replace(' ', '')+',0\n')
f.close()
# call chemprop to get scores
test_path = '\"' + 'experiments/meta_dev/' + opt.meta_dev_task + '/test_predictions.csv' + '\"'
checkpoint_path = '\"' + 'scorer_ckpts/' + opt.meta_dev_task + '/model.pt' + '\"'
preds_path = '\"' + 'experiments/meta_dev/' + opt.meta_dev_task + '/test_scores.csv' + '\"'
# in case of all mols are invalid (will produce not output file by chemprop)
# the predictions are copied into score file
cmd = 'cp {} {}'.format(test_path, preds_path)
result = os.popen(cmd)
result.close()
cmd = 'python chemprop/predict.py --test_path {} --checkpoint_path {} --preds_path {} --num_workers 0'.format(test_path, checkpoint_path, preds_path)
scorer_result = os.popen(cmd)
# logger.info('{}'.format('\n'.join(scorer_result.readlines())))
scorer_result.close()
# read score file and get score
score = read_score_csv('experiments/meta_dev/' + opt.meta_dev_task + '/test_scores.csv')
assert len(score) % opt.beam_size == 0
# report if it is the best on test
test_metrics = calculate_metrics(opt.meta_dev_task, 'dev', 'test', score)
logger.info('test metrics: ' + str(test_metrics))
test_score = test_metrics['success_rate']
if test_score > best_test_score:
best_ckpt = ckpt_path
logger.info('New best test success rate: {:.4f} by {}'.format(test_score, ckpt_path))
best_test_score = test_score
else:
logger.info('test success rate: {:.4f} by {}'.format(test_score, ckpt_path))
del model_opt
del fields
del model
gc.collect()
import onmt.opts as opts
from train_multi import main as multi_main
from train_single import main as single_main
def main(opt):
if opt.rnn_type == "SRU" and not opt.gpuid:
raise AssertionError("Using SRU requires -gpuid set.")
if torch.cuda.is_available() and not opt.gpuid:
print("WARNING: You have a CUDA device, should run with -gpuid 0")
if len(opt.gpuid) > 1:
multi_main(opt)
else:
single_main(opt)
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
description='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
opts.add_md_help_argument(PARSER)
opts.model_opts(PARSER)
opts.train_opts(PARSER)
OPT = PARSER.parse_args()
main(OPT)
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's usefull in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
vocab = checkpoint['vocab']
else:
checkpoint = None
model_opt = opt
vocab = torch.load(opt.data + '.vocab.pt')
# check for code where vocab is saved instead of fields
# (in the future this will be done in a smarter way)
if old_style_vocab(vocab):
data_type = opt.model_type
fields = load_old_vocab(vocab, data_type, dynamic_dict=opt.copy_attn)
else:
fields = vocab
# Report src and tgt vocab sizes, including for features
for side in ['src', 'tgt']:
for name, f in fields[side]:
try:
f_iter = iter(f)
except TypeError:
f_iter = [(name, f)]
for sn, sf in f_iter:
if sf.use_vocab:
logger.info(' * %s vocab size = %d' % (sn, len(sf.vocab)))
# Build model.
model = build_model(model_opt, opt, fields, checkpoint)
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
# Build optimizer.
optim = Optimizer.from_opt(model, opt, checkpoint=checkpoint)
# Build model saver
model_saver = build_model_saver(model_opt, opt, model, fields, optim)
trainer = build_trainer(opt,
device_id,
model,
fields,
optim,
model_saver=model_saver)
# this line is kind of a temporary kludge because different objects expect
# fields to have a different structure
dataset_fields = dict(chain.from_iterable(fields.values()))
train_iter = build_dataset_iter("train", dataset_fields, opt)
valid_iter = build_dataset_iter("valid",
dataset_fields,
opt,
is_train=False)
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
train_steps = opt.train_steps
if opt.single_pass and train_steps > 0:
logger.warning("Option single_pass is enabled, ignoring train_steps.")
train_steps = 0
trainer.train(train_iter,
train_steps,
save_checkpoint_steps=opt.save_checkpoint_steps,
valid_iter=valid_iter,
valid_steps=opt.valid_steps)
if opt.tensorboard:
trainer.report_manager.tensorboard_writer.close()
def build_translator(
model='forward_models/fine_tuned_model_on_liu_dataset_0p02.pt',
use_gpu=1,
batch_size=500,
report_score=False,
logger=None,
out_file=None,
log_probs_out_file=None):
opt_dict = {
'alpha': 0.0,
'beam_size': 5,
'beta': -0.0,
'block_ngram_repeat': 0,
'coverage_penalty': 'none',
'data_type': 'text',
'dump_beam': '',
'dynamic_dict': False,
'fast': True,
'gpu': use_gpu,
'ignore_when_blocking': [],
'image_channel_size': 3,
'length_penalty': 'none',
'log_file': '',
'log_probs': True,
'mask_from': '',
'max_length': 200,
'max_sent_length': None,
'min_length': 0,
'models': [model],
'n_best': 5,
'replace_unk': True,
'report_bleu': False,
'report_rouge': False,
'sample_rate': 16000,
'share_vocab': False,
'stepwise_penalty': False,
'tgt': None,
'verbose': False,
'window': 'hamming',
'window_size': 0.02,
'window_stride': 0.01
}
opt = argparse.Namespace()
for name in opt_dict.keys():
setattr(opt, name, opt_dict[name])
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
scorer = onmt.translate.GNMTGlobalScorer(opt.alpha, opt.beta,
opt.coverage_penalty,
opt.length_penalty)
kwargs = {
k: getattr(opt, k)
for k in [
"beam_size", "n_best", "max_length", "min_length",
"stepwise_penalty", "block_ngram_repeat", "ignore_when_blocking",
"dump_beam", "report_bleu", "data_type", "replace_unk", "gpu",
"verbose", "fast", "sample_rate", "window_size", "window_stride",
"window", "image_channel_size", "mask_from"
]
}
translator = Translator(model,
fields,
global_scorer=scorer,
out_file=out_file,
report_score=report_score,
copy_attn=model_opt.copy_attn,
logger=logger,
log_probs_out_file=log_probs_out_file,
**kwargs)
return translator
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's usefull in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
else:
checkpoint = None
model_opt = opt
# Peek the first dataset to determine the data_type.
# (All datasets have the same data_type).
first_dataset = next(lazily_load_dataset("train", opt))
# print("[onmt.train_single.py] first_dataset.examples[0]: {}".format(first_dataset.examples[0]))
# print("[onmt.train_single.py] first_dataset.examples[0].src[:10]: {}".format(first_dataset.examples[0].src[:10]))
print("[onmt.train_single.py] first_dataset.examples[0].src_da_label: {}".
format(first_dataset.examples[0].src_da_label))
print(
"[onmt.train_single.py] first_dataset.examples[0].__dict__.keys(): {}".
format(first_dataset.examples[0].__dict__.keys()))
data_type = first_dataset.data_type
print("[onmt.train_single.py] first_dataset.data_type: {}".format(
first_dataset.data_type))
# Load fields generated from preprocess phase.
fields = load_fields(first_dataset, opt, checkpoint)
# Report src/tgt features.
knl_features, src_features, tgt_features = _collect_report_features(fields)
for j, feat in enumerate(src_features):
logger.info(' * knl feature %d size = %d' %
(j, len(fields[feat].vocab)))
for j, feat in enumerate(src_features):
logger.info(' * src feature %d size = %d' %
(j, len(fields[feat].vocab)))
for j, feat in enumerate(tgt_features):
logger.info(' * tgt feature %d size = %d' %
(j, len(fields[feat].vocab)))
# Build model.
model = build_model(model_opt, opt, fields, checkpoint)
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
# Build optimizer.
optim = build_optim(model, opt, checkpoint)
# Build model saver
model_saver = build_model_saver(model_opt, opt, model, fields, optim)
trainer = build_trainer(opt,
device_id,
model,
fields,
optim,
data_type,
model_saver=model_saver)
def train_iter_fct():
return build_dataset_iter(lazily_load_dataset("train", opt), fields,
opt)
def valid_iter_fct():
return build_dataset_iter(lazily_load_dataset("valid", opt),
fields,
opt,
is_train=False)
# Do training.
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
opt.valid_steps)
if opt.tensorboard:
trainer.report_manager.tensorboard_writer.close()
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's usefull in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
vocab = checkpoint['vocab']
else:
checkpoint = None
model_opt = opt
vocab = torch.load(opt.data + '.vocab.pt')
# Load a shard dataset to determine the data_type.
# (All datasets have the same data_type).
# this should be refactored out of existence reasonably soon
first_dataset = torch.load(glob.glob(opt.data + '.train*.pt')[0])
data_type = first_dataset.data_type
# check for code where vocab is saved instead of fields
# (in the future this will be done in a smarter way
if old_style_vocab(vocab):
fields = load_fields_from_vocab(vocab, data_type)
else:
fields = vocab
# Report src and tgt vocab sizes, including for features
for side in ['src', 'tgt']:
for name, f in fields[side]:
if f.use_vocab:
logger.info(' * %s vocab size = %d' % (name, len(f.vocab)))
# Build model.
model = build_model(model_opt, opt, fields, checkpoint)
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
# Build optimizer.
optim = build_optim(model, opt, checkpoint)
# Build model saver
model_saver = build_model_saver(model_opt, opt, model, fields, optim)
trainer = build_trainer(opt,
device_id,
model,
fields,
optim,
data_type,
model_saver=model_saver)
# this line is kind of a temporary kludge because different objects expect
# fields to have a different structure
dataset_fields = dict(chain.from_iterable(fields.values()))
train_iter = build_dataset_iter("train", dataset_fields, opt)
valid_iter = build_dataset_iter("valid",
dataset_fields,
opt,
is_train=False)
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
trainer.train(train_iter, valid_iter, opt.train_steps, opt.valid_steps)
if opt.tensorboard:
trainer.report_manager.tensorboard_writer.close()
def main(opt):
if opt.gpuid:
raise AssertionError("gpuid is deprecated \
see world_size and gpu_ranks")
assert opt.world_size <= 1, "you don't need multi-gpu for morphology"
device_id = 0 if len(opt.gpu_ranks) == 1 else -1
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's useful in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
fields = checkpoint['vocab']
else:
checkpoint = None
model_opt = opt
fields = torch.load(opt.data + '.vocab.pt')
for key, values in fields.items():
for name, f in values:
if hasattr(f, 'use_vocab') and f.use_vocab:
logger.info(' * %s vocab size = %d' % (name, len(f.vocab)))
# Build model.
logger.info('Building model...')
model = build_model(model_opt, fields, use_gpu(opt), checkpoint)
logger.info(model)
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
# Build optimizer.
params = model.parameters()
optim_args = {"lr": opt.learning_rate}
if opt.optim == "adam":
# no need to mess with the default betas
optim_args["eps"] = 1e-9
elif opt.optim == "adagrad":
optim_args["initial_accumulator_value"] = opt.adagrad_accumulator_init
optim = getattr(torch.optim, opt.optim.title())(params, **optim_args)
print(optim)
trainer = build_trainer(opt, model_opt, device_id, model, fields, optim)
# this line is kind of a temporary kludge because different objects expect
# fields to have a different structure
dataset_fields = dict(chain.from_iterable(fields.values()))
device = "cuda" if opt.gpu_ranks else "cpu"
train_dataset = torch.load(opt.data + '.train.pt')
train_dataset.fields = dataset_fields
train_iter = OrderedIterator(
train_dataset, opt.batch_size, sort_within_batch=True,
device=device, repeat=False, shuffle=not opt.no_shuffle)
valid_dataset = torch.load(opt.data + '.valid.pt')
valid_dataset.fields = dataset_fields
valid_iter = OrderedIterator(
valid_dataset, opt.valid_batch_size, train=False,
sort_within_batch=True, device=device)
logger.info('Starting training on {}'.format(device))
trainer.train(train_iter, valid_iter, opt.epochs)
def main(opt, device_id):
opt = training_opt_postprocessing(opt, device_id)
init_logger(opt.log_file)
# Gather information related to the training script and commit version
script_path = os.path.abspath(__file__)
script_dir = os.path.dirname(os.path.dirname(script_path))
logger.info('Train script dir: %s' % script_dir)
git_commit = str(subprocess.check_output(['bash', script_dir + '/cluster_scripts/git_version.sh']))
logger.info("Git Commit: %s" % git_commit[2:-3])
# Load checkpoint if we resume from a previous training.
if opt.train_from:
# TODO: load MTL model
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
# Load default opts values then overwrite it with opts from
# the checkpoint. It's usefull in order to re-train a model
# after adding a new option (not set in checkpoint)
dummy_parser = configargparse.ArgumentParser()
opts.model_opts(dummy_parser)
default_opt = dummy_parser.parse_known_args([])[0]
model_opt = default_opt
model_opt.__dict__.update(checkpoint['opt'].__dict__)
else:
checkpoint = None
model_opt = opt
num_tasks = len(opt.data.split(','))
opt.num_tasks = num_tasks
checkpoint_list=[]
if opt.warm_model:
base_name=opt.warm_model
for task_id in range(num_tasks):
chkpt_path=base_name.replace("X",str(task_id))
if not os.path.isfile(chkpt_path):
chkpt_path = base_name.replace("X", str(0))
logger.info('Loading a checkpoint from %s' % chkpt_path)
checkpoint = torch.load(chkpt_path,
map_location=lambda storage, loc: storage)
checkpoint_list.append(checkpoint)
else:
for task_id in range(num_tasks):
checkpoint_list.append(None)
fields_list = []
data_type=None
for task_id in range(num_tasks):
# Peek the first dataset to determine the data_type.
# (All datasets have the same data_type).
first_dataset = next(lazily_load_dataset("train", opt, task_id=task_id))
data_type = first_dataset.data_type
# Load fields generated from preprocess phase.
if opt.mtl_shared_vocab and task_id > 0:
logger.info(' * vocabulary size. Same as the main task!')
fields = fields_list[0]
else:
fields = load_fields(first_dataset, opt, checkpoint_list[task_id], task_id=task_id)
# Report src/tgt features.
src_features, tgt_features = _collect_report_features(fields)
for j, feat in enumerate(src_features):
logger.info(' * (Task %d) src feature %d size = %d'
% (task_id, j, len(fields[feat].vocab)))
for j, feat in enumerate(tgt_features):
logger.info(' * (Task %) tgt feature %d size = %d'
% (task_id, j, len(fields[feat].vocab)))
fields_list.append(fields)
if opt.epochs > -1:
total_num_batch = 0
for task_id in range(num_tasks):
train_iter = build_dataset_iter(lazily_load_dataset("train", opt, task_id=task_id), fields_list[task_id], opt)
for i, batch in enumerate(train_iter):
num_batch = i
total_num_batch+=num_batch
if opt.mtl_schedule < 10:
break
num_batch = total_num_batch
opt.train_steps = (num_batch * opt.epochs) + 1
# Do the validation and save after each epoch
opt.valid_steps = num_batch
opt.save_checkpoint_steps = 1
# logger.info(opt_to_string(opt))
logger.info(opt)
# Build model(s).
models_list = []
for task_id in range(num_tasks):
if opt.mtl_fully_share and task_id > 0:
# Since we only have one model, copy the pointer to the model for all
models_list.append(models_list[0])
else:
main_model = models_list[0] if task_id > 0 else None
model = build_model(model_opt, opt, fields_list[task_id], checkpoint_list[task_id], main_model=main_model, task_id=task_id)
n_params, enc, dec = _tally_parameters(model)
logger.info('(Task %d) encoder: %d' % (task_id, enc))
logger.info('(Task %d) decoder: %d' % (task_id, dec))
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
models_list.append(model)
# combine parameters of different models and consider shared parameters just once.
def combine_named_parameters(named_params_list):
observed_params = []
for model_named_params in named_params_list:
for name, p in model_named_params:
is_observed = False
# Check whether we observed this parameter before
for param in observed_params:
if p is param:
is_observed = True
break
if not is_observed:
observed_params.append(p)
yield name, p
# Build optimizer.
optims_list = []
all_models_params=[]
for task_id in range(num_tasks):
if not opt.mtl_shared_optimizer:
optim = build_optim(models_list[task_id], opt, checkpoint)
optims_list.append(optim)
else:
all_models_params.append(models_list[task_id].named_parameters())
# Extract the list of shared parameters among the models of all tasks.
observed_params = []
shared_params = []
for task_id in range(num_tasks):
for name, p in models_list[task_id].named_parameters():
is_observed = False
# Check whether we observed this parameter before
for param in observed_params:
if p is param:
shared_params.append(name)
is_observed = True
break
if not is_observed:
observed_params.append(p)
opt.shared_params = shared_params
if opt.mtl_shared_optimizer:
optim = build_optim_mtl_params(combine_named_parameters(all_models_params), opt, checkpoint)
optims_list.append(optim)
# Build model saver
model_saver = build_mtl_model_saver(model_opt, opt, models_list, fields_list, optims_list)
trainer = build_trainer(opt, device_id, models_list, fields_list,
optims_list, data_type, model_saver=model_saver)
def train_iter_fct(task_id):
return build_dataset_iter(
lazily_load_dataset("train", opt, task_id=task_id), fields_list[task_id], opt)
def valid_iter_fct(task_id):
return build_dataset_iter(
lazily_load_dataset("valid", opt, task_id=task_id), fields_list[task_id], opt)
def meta_valid_iter_fct(task_id, is_log=False):
return build_dataset_iter(
lazily_load_dataset("meta_valid", opt, task_id=task_id, is_log=is_log), fields_list[task_id], opt)
# Do training.
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
opt.valid_steps, meta_valid_iter_fct=meta_valid_iter_fct)
if opt.tensorboard:
trainer.report_manager.tensorboard_writer.close()
def translate(opt):
out_file = codecs.open(opt.output, 'w+', 'utf-8')
if opt.gpu > -1:
torch.cuda.set_device(opt.gpu)
dummy_parser = argparse.ArgumentParser(description='train.py')
opts.model_opts(dummy_parser)
dummy_opt = dummy_parser.parse_known_args([])[0]
fields, model, model_opt = \
onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)
data = inputters.build_dataset(fields,
'text',
src_path=opt.src,
src_data_iter=None,
tgt_path=opt.tgt,
tgt_data_iter=None,
src_dir=opt.src_dir,
sample_rate='16000',
window_size=.02,
window_stride=.01,
window='hamming',
use_filter_pred=False)
device = torch.device('cuda' if opt.gpu > -1 else 'cpu')
batch_size = 1
data_iter = inputters.OrderedIterator(dataset=data,
device=device,
batch_size=batch_size,
train=False,
sort=False,
sort_within_batch=True,
shuffle=False)
pair_size = model_opt.wpe_pair_size
s_id = fields["tgt"].vocab.stoi['<s>']
if '<sgo>' in fields["tgt"].vocab.stoi:
ss_id = fields["tgt"].vocab.stoi['<sgo>']
else:
ss_id = fields['tgt'].vocab.stoi['<unk>']
if '<seos>' in fields['tgt'].vocab.stoi:
eos_id = fields['tgt'].vocab.stoi['<seos>']
else:
eos_id = fields['tgt'].vocab.stoi['</s>']
for i, batch in enumerate(data_iter):
tgt = torch.LongTensor([s_id] * batch_size + [ss_id] *
((pair_size - 1) * batch_size)).view(
pair_size,
batch_size).unsqueeze(2).to(device)
dec_state = None
src = inputters.make_features(batch, 'src', 'text')
_, src_lengths = batch.src
result = None
for _ in range(opt.max_length):
outputs, _, dec_state = model(src, tgt, src_lengths, dec_state)
scores = model.generator(outputs.view(-1, outputs.size(2)))
indices = scores.argmax(dim=1)
tgt = indices.view(pair_size, batch_size,
1) # (pair_size x batch x feat)
assert batch_size == 1
if tgt[0][0][0].item() == eos_id:
break
if result is None:
result = indices.view(pair_size, batch_size)
else:
result = torch.cat(
[result, indices.view(pair_size, batch_size)], 0)
result = result.transpose(0, 1).tolist()
for sent in result:
sent = [fields["tgt"].vocab.itos[_] for _ in sent]
sent = [_ for _ in sent if _ not in ['<blank>', '<seos>', '</s>']]
sent = ' '.join(sent)
out_file.write(sent + '\n')
print('Translated {} batches'.format(i))
out_file.close()