def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
parser.add(
'--model',
'-model',
dest='models',
metavar='MODEL',
nargs='+',
type=str,
default=[
"F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data_step_100.pt"
],
required=False,
help="模型使用得训练文件")
parser.add(
'--src',
'-src',
required=False,
default=
"F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/src-test.txt",
help="自己写的测试文件在哪里？？？？")
parser.add(
'--output',
'-output',
default=
'F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/pred.txt',
help="测试文件输出未知 改成自己得")
return parser
def translate_file(input_filename, output_filename):
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
args = f'''-model Experiments/Checkpoints/retrosynthesis_augmented_medium/retrosynthesis_aug_medium_model_step_100000.pt
-src MCTS_data/{input_filename}.txt
-output MCTS_data/{output_filename}.txt
-batch_size 128
-replace_unk
-max_length 200
-verbose
-beam_size 10
-n_best 10
-min_length 5
-gpu 0'''
opt = parser.parse_args(args)
translator = build_translator(opt, report_score=True)
src_shards = split_corpus(opt.src, opt.shard_size)
tgt_shards = repeat(None)
shard_pairs = zip(src_shards, tgt_shards)
for i, (src_shard, tgt_shard) in enumerate(shard_pairs):
scores, predictions = translator.translate(src=src_shard,
tgt=tgt_shard,
src_dir=opt.src_dir,
batch_size=opt.batch_size,
attn_debug=opt.attn_debug)
return scores, predictions
def parse_opt(opt, model_root):
argv = []
parser = ArgumentParser()
onmt.opts.model_opts(parser)
onmt.opts.translate_opts(parser)
models = opt["models"]
if not isinstance(models, (list, tuple)):
models = [models]
opt["models"] = [os.path.join(model_root, model) for model in models]
opt["src"] = "dummy_src"
for (k, v) in opt.items():
if k == "models":
argv += ["-model"]
argv += [str(model) for model in v]
elif type(v) == bool:
argv += ["-%s" % k]
else:
argv += ["-%s" % k, str(v)]
opt = parser.parse_args(argv)
ArgumentParser.validate_translate_opts(opt)
opt.cuda = opt.gpu > -1
return opt
def translate_file(input_filename, output_filename):
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
# print(opts)
args = f'''-model m16_step_44000.pt
-src source_products_16.txt
-output op_16x_4400_50_10.txt
-batch_size 128
-replace_unk
-max_length 200
-verbose
-beam_size 50
-n_best 10
-min_length 5'''
opt = parser.parse_args(args)
# print(opt.model)
translator = build_translator(opt, report_score=True)
src_shards = split_corpus(opt.src, opt.shard_size)
tgt_shards = repeat(None)
shard_pairs = zip(src_shards, tgt_shards)
for i, (src_shard, tgt_shard) in enumerate(shard_pairs):
scores, predictions = translator.translate(src=src_shard,
tgt=tgt_shard,
src_dir=opt.src_dir,
batch_size=opt.batch_size,
attn_debug=opt.attn_debug)
return scores, predictions
def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
print(parser.parse_args())
return parser
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
parser.add('--data', '-data', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data/demo',
help='Path prefix to the ".train.pt" and '
'".valid.pt" file path from preprocess.py')
parser.add('--save_model', '-save_model', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data',
help="Model filename (the model will be saved as "
"<save_model>_N.pt where N is the number "
"of steps")
parser.add('--save_checkpoint_steps', '-save_checkpoint_steps',
type=int, default=500,
help="""Save a checkpoint every X steps""")
parser.add('--train_from', '-train_from',
# default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt',
default='',
type=str,
help="If training from a checkpoint then this is the "
"path to the pretrained model's state_dict.")
# default = 100000,
parser.add('--train_steps', '-train_steps', type=int, default=100000,
help='训练多少步')
return parser
def _get_parser():
parser = ArgumentParser(description='preprocess.py')
parser.add('--train_src', '-train_src', required=False, nargs='+',
default=['F:/Project/Python/selfProject/translate_NMT/data/src-train.txt'],
help="Path(s) to the training source data")
parser.add('--train_tgt', '-train_tgt', required=False, nargs='+',
default=['F:/Project/Python/selfProject/translate_NMT/data/tgt-train.txt'],
help="Path(s) to the training target data")
parser.add('--valid_src', '-valid_src',
default='F:/Project/Python/selfProject/translate_NMT/data/src-val.txt',
help="Path to the validation source data")
parser.add('--valid_tgt', '-valid_tgt',
default='F:/Project/Python/selfProject/translate_NMT/data/tgt-val.txt',
help="Path to the validation target data")
parser.add('--save_data', '-save_data', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data/demo',
help="Output file for the prepared data")
parser.add('--num_threads', '-num_threads', type=int, default=1,
help="Number of shards to build in parallel.")
parser_init(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
opts.mmod_finetune_translate_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='run_kp_eval.py')
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='build_copy_transformer.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='translate.py')
# import ipdb; ipdb.set_trace()
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def _get_opt(self, language, method):
parser = ArgumentParser(description='summarizer.py')
if language == 'en' and method == 'bert':
config_file = 'en_bert_transformer.yml'
elif language == 'en' and method == 'conv':
config_file = 'en_conv_transformer.yml'
elif language == 'de' and method == 'bert':
config_file = 'de_bert_transformer.yml'
elif language == 'de' and method == 'conv':
config_file = 'de_conv_transformer.yml'
else:
sys.stderr.write(
f"Method '{method}' for language '{language}' is not supported."
)
#Hack to load parser arguments
prec_argv = sys.argv
sys.argv = [sys.argv[0]]
sys.argv.extend(['-config', 'config/' + config_file])
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args()
sys.argv = prec_argv
return opt
def main():
parser = ArgumentParser()
opts.config_opts(parser)
opts.model_opts(parser)
opts.global_opts(parser)
opt = parser.parse_args()
with open(os.path.join(dir_path, 'opt_data'), 'wb') as f:
pickle.dump(opt, f)
def _get_parser():
parser = ArgumentParser(description='RL_train.py')
opts.config_opts(parser)
# yida RL
opts.model_opts(parser)
# opts.train_opts(parser)
opts.translate_opts(parser)
return parser
def __init__(self):
parser = ArgumentParser()
opts.config_opts(parser)
self.opt = parser.parse_args()
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
self.mecab = Mecab.Tagger("-Owakati")
self.mecab.parce("")
def onmt_parser() -> ArgumentParser:
"""
Create the OpenNMT parser, adapted from OpenNMT-Py repo.
"""
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def build_vocabulary(ds: Dataset) -> None:
base_args = ([
"-config", f"{path.join(ds.path, 'config.yaml')}", "-n_sample", "10000"
])
parser = ArgumentParser(description='vocab.py')
dynamic_prepare_opts(parser, build_vocab_only=True)
options, unknown = parser.parse_known_args(base_args)
build_vocab_main(options)
return options, unknown
def configure_opt(self, description):
# Uses OpenNMT's ArgumentParser class to create an object
# That holds all the parameters needed to load the model
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
opt = {a.dest: a.default for a in parser._actions}
opt.update(description['opt'])
opt['models'] = [description['model']]
opt = types.SimpleNamespace(**opt)
return opt
def load_model(self, path=None):
if path is None or not os.path.exists(
os.path.abspath(os.path.join(os.getcwd(), path))):
print("No model present at the specified path : {}".format(path))
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args(["--model", path])
self.model = build_translator(opt, report_score=True)
return
def get_default_opts():
parser = ArgumentParser(description='data sample prepare')
dynamic_prepare_opts(parser)
default_opts = [
'-config', 'data/data.yaml', '-src_vocab', 'data/vocab-train.src',
'-tgt_vocab', 'data/vocab-train.tgt'
]
opt = parser.parse_known_args(default_opts)[0]
# Inject some dummy training options that may needed when build fields
opt.copy_attn = False
ArgumentParser.validate_prepare_opts(opt)
return opt
def _get_parser():
parser = ArgumentParser(description='train.py')
parser.add_argument('--teacher_model_path',
action='store',
dest='teacher_model_path',
help='the path direct to the teacher model path')
parser.add_argument("--word_sampling",
action="store",
default=False,
help="optional arg")
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def __init__(self):
# コマンドラインで指定したオプションをもとにモデルを読み込む
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
self.opt = parser.parse_args()
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
# 分かち書きのためにMeCabを使用
self.mecab = MeCab.Tagger("-Owakati")
self.mecab.parse("")
# 前回の応答を保存しておく辞書
self.prev_uttr_dict = {}
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
'''extended opts for pretrained language models'''
group = parser.add_argument_group("extended opts")
group.add('--pretrained_encoder',
'-pretrained_encoder',
default="bert",
type=str,
choices=["bert", "roberta", "xlnet"],
help="choose a pretrained language model as encoder")
return parser
def __init__(self):
# おまじない
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
self.opt = parser.parse_args(args=[
"-model", "../models/model.pt", "-src", "None", "-replace_unk",
"--beam_size", "10", "--min_length", "7", "--block_ngram_repeat",
"2"
])
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
# 単語分割用にMeCabを使用
self.mecab = MeCab.Tagger("-Owakati")
self.mecab.parse("")
def _build_translator(args):
"""
Initializes a seq2seq translator model
"""
from onmt.utils.parse import ArgumentParser
parser = ArgumentParser()
import onmt.opts as opts
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args(args=args)
ArgumentParser.validate_translate_opts(opt)
from onmt.translate.translator import build_translator
translator = build_translator(opt, report_score=False)
return translator, opt
def parse_opt(self, opt):
"""Parse the option set passed by the user using `onmt.opts`
Args:
opt (dict): Options passed by the user
Returns:
opt (argparse.Namespace): full set of options for the Translator
"""
prec_argv = sys.argv
sys.argv = sys.argv[:1]
parser = ArgumentParser()
onmt.opts.translate_opts(parser)
models = opt['models']
if not isinstance(models, (list, tuple)):
models = [models]
opt['models'] = [
os.path.join(self.model_root, model) for model in models
]
opt['src'] = "dummy_src"
for (k, v) in opt.items():
if k == 'models':
sys.argv += ['-model']
sys.argv += [str(model) for model in v]
elif type(v) == bool: # only true bool should be parsed
if v is True:
sys.argv += ['-%s' % k]
else:
sys.argv += ['-%s' % k, str(v)]
opt = parser.parse_args()
opt.alignment_heads = 8
ArgumentParser.validate_translate_opts(opt)
opt.cuda = opt.gpu > -1
sys.argv = prec_argv
return opt
def onmt_translator_builder(my_opts, s0_model_path, logger=None):
if logger is not None:
logger.info('Building ONMT translator with model from ' + s0_model_path)
else:
print('Building ONMT translator with model from ' + s0_model_path)
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
arglist = ['-model', s0_model_path] + opts_to_list(my_opts)
print(arglist)
opt = parser.parse_args(arglist)
ArgumentParser.validate_translate_opts(opt)
translator = build_translator(opt, report_score=True, logger=logger)
if logger is not None:
logger.info('Finished building ONMT translator.')
else:
print('Finished building ONMT translator.')
return translator
def translate_trained_model(n_latent,
data_path,
output_dir,
trained_model_path,
use_segments=False,
max_segments=10):
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args('')
for k, v in translate.DEFAULT_TRANSLATE_PARAMS.items():
vars(opt)[k] = v
vars(opt)['models'] = [trained_model_path]
src_path = '/'.join(data_path.split('/')[:-1]) + '/src-test.txt'
vars(opt)['src'] = src_path
output_path = '/'.join(output_dir.split('/')[:-2]) + '/preds'
vars(opt)['output_dir'] = output_path
vars(opt)['n_latent'] = n_latent
vars(opt)['use_segments'] = use_segments
vars(opt)['max_segments'] = max_segments
translate.main(opt)
def _parse_opt(opt):
prec_argv = sys.argv
sys.argv = sys.argv[:1]
parser = ArgumentParser()
onmt.opts.translate_opts(parser)
opt['src'] = "dummy_src"
opt['replace_unk'] = True
for (k, v) in opt.items():
if k == 'models':
sys.argv += ['-model']
sys.argv += [str(model) for model in v]
elif type(v) == bool:
sys.argv += ['-%s' % k]
else:
sys.argv += ['-%s' % k, str(v)]
opt = parser.parse_args()
ArgumentParser.validate_translate_opts(opt)
opt.cuda = opt.gpu > -1
sys.argv = prec_argv
return opt
