def _get_parser():
parser = ArgumentParser(description='run_kp_eval.py')
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def translate_file(input_filename, output_filename):
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
args = f'''-model Experiments/Checkpoints/retrosynthesis_augmented_medium/retrosynthesis_aug_medium_model_step_100000.pt
-src MCTS_data/{input_filename}.txt
-output MCTS_data/{output_filename}.txt
-batch_size 128
-replace_unk
-max_length 200
-verbose
-beam_size 10
-n_best 10
-min_length 5
-gpu 0'''
opt = parser.parse_args(args)
translator = build_translator(opt, report_score=True)
src_shards = split_corpus(opt.src, opt.shard_size)
tgt_shards = repeat(None)
shard_pairs = zip(src_shards, tgt_shards)
for i, (src_shard, tgt_shard) in enumerate(shard_pairs):
scores, predictions = translator.translate(src=src_shard,
tgt=tgt_shard,
src_dir=opt.src_dir,
batch_size=opt.batch_size,
attn_debug=opt.attn_debug)
return scores, predictions
def _get_parser():
parser = ArgumentParser(description='build_copy_transformer.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_opt(self, language, method):
parser = ArgumentParser(description='summarizer.py')
if language == 'en' and method == 'bert':
config_file = 'en_bert_transformer.yml'
elif language == 'en' and method == 'conv':
config_file = 'en_conv_transformer.yml'
elif language == 'de' and method == 'bert':
config_file = 'de_bert_transformer.yml'
elif language == 'de' and method == 'conv':
config_file = 'de_conv_transformer.yml'
else:
sys.stderr.write(
f"Method '{method}' for language '{language}' is not supported."
)
#Hack to load parser arguments
prec_argv = sys.argv
sys.argv = [sys.argv[0]]
sys.argv.extend(['-config', 'config/' + config_file])
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args()
sys.argv = prec_argv
return opt
def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
parser.add(
'--model',
'-model',
dest='models',
metavar='MODEL',
nargs='+',
type=str,
default=[
"F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data_step_100.pt"
],
required=False,
help="模型使用得训练文件")
parser.add(
'--src',
'-src',
required=False,
default=
"F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/src-test.txt",
help="自己写的测试文件在哪里？？？？")
parser.add(
'--output',
'-output',
default=
'F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/pred.txt',
help="测试文件输出未知 改成自己得")
return parser
def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
print(parser.parse_args())
return parser
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
parser.add('--data', '-data', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data/demo',
help='Path prefix to the ".train.pt" and '
'".valid.pt" file path from preprocess.py')
parser.add('--save_model', '-save_model', required=False,
default='F:/Project/Python/selfProject/translate_NMT/data',
help="Model filename (the model will be saved as "
"<save_model>_N.pt where N is the number "
"of steps")
parser.add('--save_checkpoint_steps', '-save_checkpoint_steps',
type=int, default=500,
help="""Save a checkpoint every X steps""")
parser.add('--train_from', '-train_from',
# default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt',
default='',
type=str,
help="If training from a checkpoint then this is the "
"path to the pretrained model's state_dict.")
# default = 100000,
parser.add('--train_steps', '-train_steps', type=int, default=100000,
help='训练多少步')
return parser
def translate_file(input_filename, output_filename):
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
# print(opts)
args = f'''-model m16_step_44000.pt
-src source_products_16.txt
-output op_16x_4400_50_10.txt
-batch_size 128
-replace_unk
-max_length 200
-verbose
-beam_size 50
-n_best 10
-min_length 5'''
opt = parser.parse_args(args)
# print(opt.model)
translator = build_translator(opt, report_score=True)
src_shards = split_corpus(opt.src, opt.shard_size)
tgt_shards = repeat(None)
shard_pairs = zip(src_shards, tgt_shards)
for i, (src_shard, tgt_shard) in enumerate(shard_pairs):
scores, predictions = translator.translate(src=src_shard,
tgt=tgt_shard,
src_dir=opt.src_dir,
batch_size=opt.batch_size,
attn_debug=opt.attn_debug)
return scores, predictions
def _get_parser():
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
opts.mmod_finetune_translate_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def _get_parser():
parser = ArgumentParser(description='translate.py')
# import ipdb; ipdb.set_trace()
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def main():
parser = ArgumentParser()
opts.config_opts(parser)
opts.model_opts(parser)
opts.global_opts(parser)
opt = parser.parse_args()
with open(os.path.join(dir_path, 'opt_data'), 'wb') as f:
pickle.dump(opt, f)
def _get_parser():
parser = ArgumentParser(description='RL_train.py')
opts.config_opts(parser)
# yida RL
opts.model_opts(parser)
# opts.train_opts(parser)
opts.translate_opts(parser)
return parser
def __init__(self):
parser = ArgumentParser()
opts.config_opts(parser)
self.opt = parser.parse_args()
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
self.mecab = Mecab.Tagger("-Owakati")
self.mecab.parce("")
def onmt_parser() -> ArgumentParser:
"""
Create the OpenNMT parser, adapted from OpenNMT-Py repo.
"""
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
return parser
def configure_opt(self, description):
# Uses OpenNMT's ArgumentParser class to create an object
# That holds all the parameters needed to load the model
parser = ArgumentParser(description='translation')
opts.config_opts(parser)
opts.translate_opts(parser)
opt = {a.dest: a.default for a in parser._actions}
opt.update(description['opt'])
opt['models'] = [description['model']]
opt = types.SimpleNamespace(**opt)
return opt
def parse_args():
parser = configargparse.ArgumentParser(
description='preprocess.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.config_opts(parser) #add the config into the opts
opts.preprocess_opts(parser) #add the argument needed in the project
opt = parser.parse_args() #read the arugument
torch.manual_seed(opt.seed)
return opt
def parse_args():
parser = configargparse.ArgumentParser(
description='preprocess.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.config_opts(parser)
opts.preprocess_opts(parser)
opt = parser.parse_args()
torch.manual_seed(opt.seed)
return opt
def load_model(self, path=None):
if path is None or not os.path.exists(
os.path.abspath(os.path.join(os.getcwd(), path))):
print("No model present at the specified path : {}".format(path))
parser = ArgumentParser(description='translate.py')
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args(["--model", path])
self.model = build_translator(opt, report_score=True)
return
def parse_args():
parser = configargparse.ArgumentParser(
description='train.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.general_opts(parser)
opts.config_opts(parser)
opts.add_md_help_argument(parser)
opts.model_opts(parser)
opts.train_opts(parser)
opt = parser.parse_args()
return opt
def __init__(self):
# コマンドラインで指定したオプションをもとにモデルを読み込む
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
self.opt = parser.parse_args()
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
# 分かち書きのためにMeCabを使用
self.mecab = MeCab.Tagger("-Owakati")
self.mecab.parse("")
# 前回の応答を保存しておく辞書
self.prev_uttr_dict = {}
def _get_parser():
parser = ArgumentParser(description='train.py')
parser.add_argument('--teacher_model_path',
action='store',
dest='teacher_model_path',
help='the path direct to the teacher model path')
parser.add_argument("--word_sampling",
action="store",
default=False,
help="optional arg")
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
return parser
def __init__(self):
# おまじない
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
self.opt = parser.parse_args(args=[
"-model", "../models/model.pt", "-src", "None", "-replace_unk",
"--beam_size", "10", "--min_length", "7", "--block_ngram_repeat",
"2"
])
ArgumentParser.validate_translate_opts(self.opt)
self.translator = build_translator(self.opt, report_score=True)
# 単語分割用にMeCabを使用
self.mecab = MeCab.Tagger("-Owakati")
self.mecab.parse("")
def parse_args():
parser = configargparse.ArgumentParser(
description='preprocess.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
# 预处理所需要的全部参数均在opts模块中
opts.config_opts(parser)
opts.preprocess_opts(parser)
opt = parser.parse_args()
# 指定一个随机数会使得每次按顺序使用rand...产生的数值均相同
# 不知道在不同机器上是否相同, 如果相同的话结果便可以复现
torch.manual_seed(opt.seed)
return opt
def _get_parser():
parser = ArgumentParser(description='train.py')
opts.config_opts(parser)
opts.model_opts(parser)
opts.train_opts(parser)
'''extended opts for pretrained language models'''
group = parser.add_argument_group("extended opts")
group.add('--pretrained_encoder',
'-pretrained_encoder',
default="bert",
type=str,
choices=["bert", "roberta", "xlnet"],
help="choose a pretrained language model as encoder")
return parser
def parse_args():
""" Parsing arguments """
parser = configargparse.ArgumentParser(
description='preprocess.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.config_opts(parser)
opts.add_md_help_argument(parser)
opts.preprocess_opts(parser)
opt = parser.parse_args()
torch.manual_seed(opt.seed)
check_existing_pt_files(opt)
return opt
def _build_translator(args):
"""
Initializes a seq2seq translator model
"""
from onmt.utils.parse import ArgumentParser
parser = ArgumentParser()
import onmt.opts as opts
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args(args=args)
ArgumentParser.validate_translate_opts(opt)
from onmt.translate.translator import build_translator
translator = build_translator(opt, report_score=False)
return translator, opt
def __init__(self, gpu):
parser = configargparse.ArgumentParser(
description='translate.py',
config_file_parser_class=configargparse.YAMLConfigFileParser,
formatter_class=configargparse.ArgumentDefaultsHelpFormatter)
opts.config_opts(parser)
opts.add_md_help_argument(parser)
opts.translate_opts(parser)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
if len(gpu) > 1:
print('do not try hacking')
exit()
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)
sys.argv = ["python"]
opt = parser.parse_args()
opt.models = [
'./modules/multi_summ/dataset_m2s2/korean_bert_8_single_new_economy_add_cls_sep_segment_eos_penalty_new_bloom_step_25000.pt'
]
opt.segment = True
opt.batch_size = 8
opt.beam_size = 10
opt.src = '.1'
opt.output = '.1'
opt.verbose = True
opt.stepwise_penalty = True
opt.coverage_penalty = 'sumarry'
opt.beta = 5
opt.length_penalty = 'wu'
opt.alpha = 0.9
opt.block_ngram_repeat = 3
opt.ignore_when_blocking = [".", "</t", "<t>", ",_", "%"]
opt.max_length = 300
opt.min_length = 35
opt.gpu = 0
opt.segment = True
logger = init_logger(opt.log_file)
self.translator = build_translator(opt, report_score=True)
def onmt_translator_builder(my_opts, s0_model_path, logger=None):
if logger is not None:
logger.info('Building ONMT translator with model from ' + s0_model_path)
else:
print('Building ONMT translator with model from ' + s0_model_path)
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
arglist = ['-model', s0_model_path] + opts_to_list(my_opts)
print(arglist)
opt = parser.parse_args(arglist)
ArgumentParser.validate_translate_opts(opt)
translator = build_translator(opt, report_score=True, logger=logger)
if logger is not None:
logger.info('Finished building ONMT translator.')
else:
print('Finished building ONMT translator.')
return translator
def translate_trained_model(n_latent,
data_path,
output_dir,
trained_model_path,
use_segments=False,
max_segments=10):
parser = ArgumentParser()
opts.config_opts(parser)
opts.translate_opts(parser)
opt = parser.parse_args('')
for k, v in translate.DEFAULT_TRANSLATE_PARAMS.items():
vars(opt)[k] = v
vars(opt)['models'] = [trained_model_path]
src_path = '/'.join(data_path.split('/')[:-1]) + '/src-test.txt'
vars(opt)['src'] = src_path
output_path = '/'.join(output_dir.split('/')[:-2]) + '/preds'
vars(opt)['output_dir'] = output_path
vars(opt)['n_latent'] = n_latent
vars(opt)['use_segments'] = use_segments
vars(opt)['max_segments'] = max_segments
translate.main(opt)
def _get_parser():
parser = ArgumentParser(description='preprocess.py')
opts.config_opts(parser)
opts.preprocess_opts(parser)
return parser
