def _get_parser(): parser = ArgumentParser(description='run_kp_eval.py') opts.config_opts(parser) opts.translate_opts(parser) return parser
def translate_file(input_filename, output_filename): parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) args = f'''-model Experiments/Checkpoints/retrosynthesis_augmented_medium/retrosynthesis_aug_medium_model_step_100000.pt -src MCTS_data/{input_filename}.txt -output MCTS_data/{output_filename}.txt -batch_size 128 -replace_unk -max_length 200 -verbose -beam_size 10 -n_best 10 -min_length 5 -gpu 0''' opt = parser.parse_args(args) translator = build_translator(opt, report_score=True) src_shards = split_corpus(opt.src, opt.shard_size) tgt_shards = repeat(None) shard_pairs = zip(src_shards, tgt_shards) for i, (src_shard, tgt_shard) in enumerate(shard_pairs): scores, predictions = translator.translate(src=src_shard, tgt=tgt_shard, src_dir=opt.src_dir, batch_size=opt.batch_size, attn_debug=opt.attn_debug) return scores, predictions
def _get_parser(): parser = ArgumentParser(description='build_copy_transformer.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def _get_opt(self, language, method): parser = ArgumentParser(description='summarizer.py') if language == 'en' and method == 'bert': config_file = 'en_bert_transformer.yml' elif language == 'en' and method == 'conv': config_file = 'en_conv_transformer.yml' elif language == 'de' and method == 'bert': config_file = 'de_bert_transformer.yml' elif language == 'de' and method == 'conv': config_file = 'de_conv_transformer.yml' else: sys.stderr.write( f"Method '{method}' for language '{language}' is not supported." ) #Hack to load parser arguments prec_argv = sys.argv sys.argv = [sys.argv[0]] sys.argv.extend(['-config', 'config/' + config_file]) opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args() sys.argv = prec_argv return opt
def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) parser.add( '--model', '-model', dest='models', metavar='MODEL', nargs='+', type=str, default=[ "F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data_step_100.pt" ], required=False, help="模型使用得训练文件") parser.add( '--src', '-src', required=False, default= "F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/src-test.txt", help="自己写的测试文件在哪里????") parser.add( '--output', '-output', default= 'F:/Project/Python/selfProject/translate_NMT/transflate_NMT/data/pred.txt', help="测试文件输出未知 改成自己得") return parser
def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) print(parser.parse_args()) return parser
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) parser.add('--data', '-data', required=False, default='F:/Project/Python/selfProject/translate_NMT/data/demo', help='Path prefix to the ".train.pt" and ' '".valid.pt" file path from preprocess.py') parser.add('--save_model', '-save_model', required=False, default='F:/Project/Python/selfProject/translate_NMT/data', help="Model filename (the model will be saved as " "<save_model>_N.pt where N is the number " "of steps") parser.add('--save_checkpoint_steps', '-save_checkpoint_steps', type=int, default=500, help="""Save a checkpoint every X steps""") parser.add('--train_from', '-train_from', # default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt', default='', type=str, help="If training from a checkpoint then this is the " "path to the pretrained model's state_dict.") # default = 100000, parser.add('--train_steps', '-train_steps', type=int, default=100000, help='训练多少步') return parser
def translate_file(input_filename, output_filename): parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) # print(opts) args = f'''-model m16_step_44000.pt -src source_products_16.txt -output op_16x_4400_50_10.txt -batch_size 128 -replace_unk -max_length 200 -verbose -beam_size 50 -n_best 10 -min_length 5''' opt = parser.parse_args(args) # print(opt.model) translator = build_translator(opt, report_score=True) src_shards = split_corpus(opt.src, opt.shard_size) tgt_shards = repeat(None) shard_pairs = zip(src_shards, tgt_shards) for i, (src_shard, tgt_shard) in enumerate(shard_pairs): scores, predictions = translator.translate(src=src_shard, tgt=tgt_shard, src_dir=opt.src_dir, batch_size=opt.batch_size, attn_debug=opt.attn_debug) return scores, predictions
def _get_parser(): parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) opts.mmod_finetune_translate_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def _get_parser(): parser = ArgumentParser(description='translate.py') # import ipdb; ipdb.set_trace() opts.config_opts(parser) opts.translate_opts(parser) return parser
def main(): parser = ArgumentParser() opts.config_opts(parser) opts.model_opts(parser) opts.global_opts(parser) opt = parser.parse_args() with open(os.path.join(dir_path, 'opt_data'), 'wb') as f: pickle.dump(opt, f)
def _get_parser(): parser = ArgumentParser(description='RL_train.py') opts.config_opts(parser) # yida RL opts.model_opts(parser) # opts.train_opts(parser) opts.translate_opts(parser) return parser
def __init__(self): parser = ArgumentParser() opts.config_opts(parser) self.opt = parser.parse_args() ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) self.mecab = Mecab.Tagger("-Owakati") self.mecab.parce("")
def onmt_parser() -> ArgumentParser: """ Create the OpenNMT parser, adapted from OpenNMT-Py repo. """ parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) return parser
def configure_opt(self, description): # Uses OpenNMT's ArgumentParser class to create an object # That holds all the parameters needed to load the model parser = ArgumentParser(description='translation') opts.config_opts(parser) opts.translate_opts(parser) opt = {a.dest: a.default for a in parser._actions} opt.update(description['opt']) opt['models'] = [description['model']] opt = types.SimpleNamespace(**opt) return opt
def parse_args(): parser = configargparse.ArgumentParser( description='preprocess.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) opts.config_opts(parser) #add the config into the opts opts.preprocess_opts(parser) #add the argument needed in the project opt = parser.parse_args() #read the arugument torch.manual_seed(opt.seed) return opt
def parse_args(): parser = configargparse.ArgumentParser( description='preprocess.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) opts.config_opts(parser) opts.preprocess_opts(parser) opt = parser.parse_args() torch.manual_seed(opt.seed) return opt
def load_model(self, path=None): if path is None or not os.path.exists( os.path.abspath(os.path.join(os.getcwd(), path))): print("No model present at the specified path : {}".format(path)) parser = ArgumentParser(description='translate.py') opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args(["--model", path]) self.model = build_translator(opt, report_score=True) return
def parse_args(): parser = configargparse.ArgumentParser( description='train.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) opts.general_opts(parser) opts.config_opts(parser) opts.add_md_help_argument(parser) opts.model_opts(parser) opts.train_opts(parser) opt = parser.parse_args() return opt
def __init__(self): # コマンドラインで指定したオプションをもとにモデルを読み込む parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) self.opt = parser.parse_args() ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) # 分かち書きのためにMeCabを使用 self.mecab = MeCab.Tagger("-Owakati") self.mecab.parse("") # 前回の応答を保存しておく辞書 self.prev_uttr_dict = {}
def _get_parser(): parser = ArgumentParser(description='train.py') parser.add_argument('--teacher_model_path', action='store', dest='teacher_model_path', help='the path direct to the teacher model path') parser.add_argument("--word_sampling", action="store", default=False, help="optional arg") opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) return parser
def __init__(self): # おまじない parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) self.opt = parser.parse_args(args=[ "-model", "../models/model.pt", "-src", "None", "-replace_unk", "--beam_size", "10", "--min_length", "7", "--block_ngram_repeat", "2" ]) ArgumentParser.validate_translate_opts(self.opt) self.translator = build_translator(self.opt, report_score=True) # 単語分割用にMeCabを使用 self.mecab = MeCab.Tagger("-Owakati") self.mecab.parse("")
def parse_args(): parser = configargparse.ArgumentParser( description='preprocess.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) # 预处理所需要的全部参数均在opts模块中 opts.config_opts(parser) opts.preprocess_opts(parser) opt = parser.parse_args() # 指定一个随机数会使得每次按顺序使用rand...产生的数值均相同 # 不知道在不同机器上是否相同, 如果相同的话结果便可以复现 torch.manual_seed(opt.seed) return opt
def _get_parser(): parser = ArgumentParser(description='train.py') opts.config_opts(parser) opts.model_opts(parser) opts.train_opts(parser) '''extended opts for pretrained language models''' group = parser.add_argument_group("extended opts") group.add('--pretrained_encoder', '-pretrained_encoder', default="bert", type=str, choices=["bert", "roberta", "xlnet"], help="choose a pretrained language model as encoder") return parser
def parse_args(): """ Parsing arguments """ parser = configargparse.ArgumentParser( description='preprocess.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) opts.config_opts(parser) opts.add_md_help_argument(parser) opts.preprocess_opts(parser) opt = parser.parse_args() torch.manual_seed(opt.seed) check_existing_pt_files(opt) return opt
def _build_translator(args): """ Initializes a seq2seq translator model """ from onmt.utils.parse import ArgumentParser parser = ArgumentParser() import onmt.opts as opts opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args(args=args) ArgumentParser.validate_translate_opts(opt) from onmt.translate.translator import build_translator translator = build_translator(opt, report_score=False) return translator, opt
def __init__(self, gpu): parser = configargparse.ArgumentParser( description='translate.py', config_file_parser_class=configargparse.YAMLConfigFileParser, formatter_class=configargparse.ArgumentDefaultsHelpFormatter) opts.config_opts(parser) opts.add_md_help_argument(parser) opts.translate_opts(parser) os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" if len(gpu) > 1: print('do not try hacking') exit() os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) sys.argv = ["python"] opt = parser.parse_args() opt.models = [ './modules/multi_summ/dataset_m2s2/korean_bert_8_single_new_economy_add_cls_sep_segment_eos_penalty_new_bloom_step_25000.pt' ] opt.segment = True opt.batch_size = 8 opt.beam_size = 10 opt.src = '.1' opt.output = '.1' opt.verbose = True opt.stepwise_penalty = True opt.coverage_penalty = 'sumarry' opt.beta = 5 opt.length_penalty = 'wu' opt.alpha = 0.9 opt.block_ngram_repeat = 3 opt.ignore_when_blocking = [".", "</t", "<t>", ",_", "%"] opt.max_length = 300 opt.min_length = 35 opt.gpu = 0 opt.segment = True logger = init_logger(opt.log_file) self.translator = build_translator(opt, report_score=True)
def onmt_translator_builder(my_opts, s0_model_path, logger=None): if logger is not None: logger.info('Building ONMT translator with model from ' + s0_model_path) else: print('Building ONMT translator with model from ' + s0_model_path) parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) arglist = ['-model', s0_model_path] + opts_to_list(my_opts) print(arglist) opt = parser.parse_args(arglist) ArgumentParser.validate_translate_opts(opt) translator = build_translator(opt, report_score=True, logger=logger) if logger is not None: logger.info('Finished building ONMT translator.') else: print('Finished building ONMT translator.') return translator
def translate_trained_model(n_latent, data_path, output_dir, trained_model_path, use_segments=False, max_segments=10): parser = ArgumentParser() opts.config_opts(parser) opts.translate_opts(parser) opt = parser.parse_args('') for k, v in translate.DEFAULT_TRANSLATE_PARAMS.items(): vars(opt)[k] = v vars(opt)['models'] = [trained_model_path] src_path = '/'.join(data_path.split('/')[:-1]) + '/src-test.txt' vars(opt)['src'] = src_path output_path = '/'.join(output_dir.split('/')[:-2]) + '/preds' vars(opt)['output_dir'] = output_path vars(opt)['n_latent'] = n_latent vars(opt)['use_segments'] = use_segments vars(opt)['max_segments'] = max_segments translate.main(opt)
def _get_parser(): parser = ArgumentParser(description='preprocess.py') opts.config_opts(parser) opts.preprocess_opts(parser) return parser