def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) parser.add( '--model', '-model', dest='models', metavar='MODEL', nargs='+', type=str, default=[ "F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data_step_100.pt" ], required=False, help="模型使用得训练文件") parser.add( '--src', '-src', required=False, default= "F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/src-test.txt", help="自己写的测试文件在哪里????") parser.add( '--output', '-output', default= 'F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/pred.txt', help="测试文件输出未知 改成自己得") return parser
def translate_file(input_filename, output_filename): parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) args = f'''-model Experiments/Checkpoints/retrosynthesis_augmented_medium/retrosynthesis_aug_medium_model_step_100000.pt -src MCTS_data/{input_filename}.txt -output MCTS_data/{output_filename}.txt -batch_size 128 -replace_unk -max_length 200 -verbose -beam_size 10 -n_best 10 -min_length 5 -gpu 0''' opt = parser.parse_args(args) translator = build_translator(opt, report_score=True) src_shards = split_corpus(opt.src, opt.shard_size) tgt_shards = repeat(None) shard_pairs = zip(src_shards, tgt_shards) for i, (src_shard, tgt_shard) in enumerate(shard_pairs): scores, predictions = translator.translate(src=src_shard, tgt=tgt_shard, src_dir=opt.src_dir, batch_size=opt.batch_size, attn_debug=opt.attn_debug) return scores, predictions
def parse_opt(opt, model_root): argv = [] parser = ArgumentParser() onmt.opts.model_opts(parser) onmt.opts.translate_opts(parser) models = opt["models"] if not isinstance(models, (list, tuple)): models = [models] opt["models"] = [os.path.join(model_root, model) for model in models] opt["src"] = "dummy_src" for (k, v) in opt.items(): if k == "models": argv += ["-model"] argv += [str(model) for model in v] elif type(v) == bool: argv += ["-%s" % k] else: argv += ["-%s" % k, str(v)] opt = parser.parse_args(argv) ArgumentParser.validate_translate_opts(opt) opt.cuda = opt.gpu > -1 return opt
def translate_file(input_filename, output_filename): parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) # print(opts) args = f'''-model m16_step_44000.pt -src source_products_16.txt -output op_16x_4400_50_10.txt -batch_size 128 -replace_unk -max_length 200 -verbose -beam_size 50 -n_best 10 -min_length 5''' opt = parser.parse_args(args) # print(opt.model) translator = build_translator(opt, report_score=True) src_shards = split_corpus(opt.src, opt.shard_size) tgt_shards = repeat(None) shard_pairs = zip(src_shards, tgt_shards) for i, (src_shard, tgt_shard) in enumerate(shard_pairs): scores, predictions = translator.translate(src=src_shard, tgt=tgt_shard, src_dir=opt.src_dir, batch_size=opt.batch_size, attn_debug=opt.attn_debug) return scores, predictions
def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) print(parser.parse_args()) return parser
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) parser.add('--data', '-data', required=False, default='F:/Project/Python/selfProject/translate_NMT/data/demo', help='Path prefix to the ".train.pt" and ' '".valid.pt" file path from preprocess.py') parser.add('--save_model', '-save_model', required=False, default='F:/Project/Python/selfProject/translate_NMT/data', help="Model filename (the model will be saved as " "<save_model>_N.pt where N is the number " "of steps") parser.add('--save_checkpoint_steps', '-save_checkpoint_steps', type=int, default=500, help="""Save a checkpoint every X steps""") parser.add('--train_from', '-train_from', # default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt', default='', type=str, help="If training from a checkpoint then this is the " "path to the pretrained model's state_dict.") # default = 100000, parser.add('--train_steps', '-train_steps', type=int, default=100000, help='训练多少步') return parser
def _get_parser(): parser = ArgumentParser(description='preprocess.py') parser.add('--train_src', '-train_src', required=False, nargs='+', default=['F:/Project/Python/selfProject/translate_NMT/data/src-train.txt'], help="Path(s) to the training source data") parser.add('--train_tgt', '-train_tgt', required=False, nargs='+', default=['F:/Project/Python/selfProject/translate_NMT/data/tgt-train.txt'], help="Path(s) to the training target data") parser.add('--valid_src', '-valid_src', default='F:/Project/Python/selfProject/translate_NMT/data/src-val.txt', help="Path to the validation source data") parser.add('--valid_tgt', '-valid_tgt', default='F:/Project/Python/selfProject/translate_NMT/data/tgt-val.txt', help="Path to the validation target data") parser.add('--save_data', '-save_data', required=False, default='F:/Project/Python/selfProject/translate_NMT/data/demo', help="Output file for the prepared data") parser.add('--num_threads', '-num_threads', type=int, default=1, help="Number of shards to build in parallel.") parser_init(parser) return parser
def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) opts.mmod_finetune_translate_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='run_kp_eval.py') opts.config_opts(parser) opts.translate_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='build_copy_transformer.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='translate.py') # import ipdb; ipdb.set_trace() opts.config_opts(parser) opts.translate_opts(parser) return parser
def _get_opt(self, language, method): parser = ArgumentParser(description='summarizer.py') if language == 'en' and method == 'bert': config_file = 'en_bert_transformer.yml' elif language == 'en' and method == 'conv': config_file = 'en_conv_transformer.yml' elif language == 'de' and method == 'bert': config_file = 'de_bert_transformer.yml' elif language == 'de' and method == 'conv': config_file = 'de_conv_transformer.yml' else: sys.stderr.write( f"Method '{method}' for language '{language}' is not supported." ) #Hack to load parser arguments prec_argv = sys.argv sys.argv = [sys.argv[0]] sys.argv.extend(['-config', 'config/' + config_file]) opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args() sys.argv = prec_argv return opt
def main(): parser = ArgumentParser() opts.config_opts(parser) opts.model_opts(parser) opts.global_opts(parser) opt = parser.parse_args() with open(os.path.join(dir_path, 'opt_data'), 'wb') as f: pickle.dump(opt, f)
def _get_parser(): parser = ArgumentParser(description='RL_train.py') opts.config_opts(parser) # yida RL opts.model_opts(parser) # opts.train_opts(parser) opts.translate_opts(parser) return parser
def __init__(self): parser = ArgumentParser() opts.config_opts(parser) self.opt = parser.parse_args() ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) self.mecab = Mecab.Tagger("-Owakati") self.mecab.parce("")
def onmt_parser() -> ArgumentParser: """ Create the OpenNMT parser, adapted from OpenNMT-Py repo. """ parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) return parser
def build_vocabulary(ds: Dataset) -> None: base_args = ([ "-config", f"{path.join(ds.path, 'config.yaml')}", "-n_sample", "10000" ]) parser = ArgumentParser(description='vocab.py') dynamic_prepare_opts(parser, build_vocab_only=True) options, unknown = parser.parse_known_args(base_args) build_vocab_main(options) return options, unknown
def configure_opt(self, description): # Uses OpenNMT's ArgumentParser class to create an object # That holds all the parameters needed to load the model parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) opt = {a.dest: a.default for a in parser._actions} opt.update(description['opt']) opt['models'] = [description['model']] opt = types.SimpleNamespace(**opt) return opt
def load_model(self, path=None): if path is None or not os.path.exists( os.path.abspath(os.path.join(os.getcwd(), path))): print("No model present at the specified path : {}".format(path)) parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args(["--model", path]) self.model = build_translator(opt, report_score=True) return
def get_default_opts(): parser = ArgumentParser(description='data sample prepare') dynamic_prepare_opts(parser) default_opts = [ '-config', 'data/data.yaml', '-src_vocab', 'data/vocab-train.src', '-tgt_vocab', 'data/vocab-train.tgt' ] opt = parser.parse_known_args(default_opts)[0] # Inject some dummy training options that may needed when build fields opt.copy_attn = False ArgumentParser.validate_prepare_opts(opt) return opt
def _get_parser(): parser = ArgumentParser(description='train.py') parser.add_argument('--teacher_model_path', action='store', dest='teacher_model_path', help='the path direct to the teacher model path') parser.add_argument("--word_sampling", action="store", default=False, help="optional arg") opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def __init__(self): # コマンドラインで指定したオプションをもとにモデルを読み込む parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) self.opt = parser.parse_args() ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) # 分かち書きのためにMeCabを使用 self.mecab = MeCab.Tagger("-Owakati") self.mecab.parse("") # 前回の応答を保存しておく辞書 self.prev_uttr_dict = {}
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) '''extended opts for pretrained language models''' group = parser.add_argument_group("extended opts") group.add('--pretrained_encoder', '-pretrained_encoder', default="bert", type=str, choices=["bert", "roberta", "xlnet"], help="choose a pretrained language model as encoder") return parser
def __init__(self): # おまじない parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) self.opt = parser.parse_args(args=[ "-model", "../models/model.pt", "-src", "None", "-replace_unk", "--beam_size", "10", "--min_length", "7", "--block_ngram_repeat", "2" ]) ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) # 単語分割用にMeCabを使用 self.mecab = MeCab.Tagger("-Owakati") self.mecab.parse("")
def _build_translator(args): """ Initializes a seq2seq translator model """ from onmt.utils.parse import ArgumentParser parser = ArgumentParser() import onmt.opts as opts opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args(args=args) ArgumentParser.validate_translate_opts(opt) from onmt.translate.translator import build_translator translator = build_translator(opt, report_score=False) return translator, opt
def parse_opt(self, opt): """Parse the option set passed by the user using `onmt.opts` Args: opt (dict): Options passed by the user Returns: opt (argparse.Namespace): full set of options for the Translator """ prec_argv = sys.argv sys.argv = sys.argv[:1] parser = ArgumentParser() onmt.opts.translate_opts(parser) models = opt['models'] if not isinstance(models, (list, tuple)): models = [models] opt['models'] = [ os.path.join(self.model_root, model) for model in models ] opt['src'] = "dummy_src" for (k, v) in opt.items(): if k == 'models': sys.argv += ['-model'] sys.argv += [str(model) for model in v] elif type(v) == bool: # only true bool should be parsed if v is True: sys.argv += ['-%s' % k] else: sys.argv += ['-%s' % k, str(v)] opt = parser.parse_args() opt.alignment_heads = 8 ArgumentParser.validate_translate_opts(opt) opt.cuda = opt.gpu > -1 sys.argv = prec_argv return opt
def onmt_translator_builder(my_opts, s0_model_path, logger=None): if logger is not None: logger.info('Building ONMT translator with model from ' + s0_model_path) else: print('Building ONMT translator with model from ' + s0_model_path) parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) arglist = ['-model', s0_model_path] + opts_to_list(my_opts) print(arglist) opt = parser.parse_args(arglist) ArgumentParser.validate_translate_opts(opt) translator = build_translator(opt, report_score=True, logger=logger) if logger is not None: logger.info('Finished building ONMT translator.') else: print('Finished building ONMT translator.') return translator
def translate_trained_model(n_latent, data_path, output_dir, trained_model_path, use_segments=False, max_segments=10): parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args('') for k, v in translate.DEFAULT_TRANSLATE_PARAMS.items(): vars(opt)[k] = v vars(opt)['models'] = [trained_model_path] src_path = '/'.join(data_path.split('/')[:-1]) + '/src-test.txt' vars(opt)['src'] = src_path output_path = '/'.join(output_dir.split('/')[:-2]) + '/preds' vars(opt)['output_dir'] = output_path vars(opt)['n_latent'] = n_latent vars(opt)['use_segments'] = use_segments vars(opt)['max_segments'] = max_segments translate.main(opt)
def _parse_opt(opt): prec_argv = sys.argv sys.argv = sys.argv[:1] parser = ArgumentParser() onmt.opts.translate_opts(parser) opt['src'] = "dummy_src" opt['replace_unk'] = True for (k, v) in opt.items(): if k == 'models': sys.argv += ['-model'] sys.argv += [str(model) for model in v] elif type(v) == bool: sys.argv += ['-%s' % k] else: sys.argv += ['-%s' % k, str(v)] opt = parser.parse_args() ArgumentParser.validate_translate_opts(opt) opt.cuda = opt.gpu > -1 sys.argv = prec_argv return opt