Example #1
0
def _get_parser():
    parser = ArgumentParser(description='train.py')

    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)
    parser.add('--data', '-data', required=False,
               default='F:/Project/Python/selfProject/translate_NMT/data/demo',
               help='Path prefix to the ".train.pt" and '
                    '".valid.pt" file path from preprocess.py')

    parser.add('--save_model', '-save_model', required=False,
               default='F:/Project/Python/selfProject/translate_NMT/data',
               help="Model filename (the model will be saved as "
                    "<save_model>_N.pt where N is the number "
                    "of steps")

    parser.add('--save_checkpoint_steps', '-save_checkpoint_steps',
               type=int, default=500,
               help="""Save a checkpoint every X steps""")

    parser.add('--train_from', '-train_from',
               # default='F:/Project/Python/selfProject/translate_NMT/data/demo-model_step_150.pt',
               default='',
               type=str,
               help="If training from a checkpoint then this is the "
                    "path to the pretrained model's state_dict.")

    # default = 100000,
    parser.add('--train_steps', '-train_steps', type=int, default=100000,
               help='训练多少步')
    return parser
Example #2
0
def build_translator(opt, report_score=True, logger=None, out_file=None):
    if out_file is None:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')

    dummy_parser = configargparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    load_test_model = onmt.decoders.ensemble.load_test_model \
        if len(opt.models) > 1 else onmt.model_builder.load_test_model
    fields, model, model_opt = load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer.from_opt(opt)

    translator = Translator(
        model,
        fields,
        opt,
        model_opt,
        global_scorer=scorer,
        out_file=out_file,
        report_score=report_score,
        logger=logger
    )
    return translator
Example #3
0
def build_translator(opt, report_score=True, logger=None, out_file=None, use_output=True):
    if out_file is None and use_output:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')

    if opt.gpu > -1:
        torch.cuda.set_device(opt.gpu)

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    fields, model, model_opt = \
        onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
                                             opt.beta,
                                             opt.coverage_penalty,
                                             opt.length_penalty)

    kwargs = {k: getattr(opt, k)
              for k in ["beam_size", "n_best", "max_length", "min_length",
                        "stepwise_penalty", "block_ngram_repeat",
                        "ignore_when_blocking", "dump_beam", "dump_layers",
                        "report_bleu", "data_type", "replace_unk", "gpu",
                        "verbose"]}

    translator = Translator(model, fields, global_scorer=scorer,
                            out_file=out_file, report_score=report_score,
                            copy_attn=model_opt.copy_attn, logger=logger,
                            **kwargs)
    return translator
Example #4
0
def build_translator(opt):
    dummy_parser = configargparse.ArgumentParser(description='translate.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]
    fields, model = load_test_model(opt, dummy_opt.__dict__)
    translator = Translator(model, fields, opt)
    return translator
Example #5
0
def main(opt, device_id):
  opt = training_opt_postprocessing(opt, device_id)
  init_logger(opt.log_file)
  # Load checkpoint if we resume from a previous training.
  if opt.train_from:
    logger.info('Loading checkpoint from %s' % opt.train_from)
    checkpoint = torch.load(opt.train_from,
                            map_location=lambda storage, loc: storage)

    # Load default opts values then overwrite it with opts from
    # the checkpoint. It's usefull in order to re-train a model
    # after adding a new option (not set in checkpoint)
    dummy_parser = configargparse.ArgumentParser()
    opts.model_opts(dummy_parser)
    default_opt = dummy_parser.parse_known_args([])[0]

    model_opt = default_opt
    model_opt.__dict__.update(checkpoint['opt'].__dict__)
  else:
    checkpoint = None
    model_opt = opt

  # Load fields generated from preprocess phase.
  fields = load_fields(opt, checkpoint)

  # Build model.
  model = build_model(model_opt, opt, fields, checkpoint)
  n_params, enc, dec = _tally_parameters(model)
  logger.info('encoder: %d' % enc)
  logger.info('decoder: %d' % dec)
  logger.info('* number of parameters: %d' % n_params)
  _check_save_model_path(opt)

  # Build optimizer.
  optim = build_optim(model, opt, checkpoint)

  # Build model saver
  model_saver = build_model_saver(model_opt, opt, model, fields, optim)

  trainer = build_trainer(opt, device_id, model, fields,
                          optim, model_saver=model_saver)

  def train_iter_fct(): 
    return build_dataset_iter(
      load_dataset("train", opt), fields, opt)

  def valid_iter_fct(): 
    return build_dataset_iter(
      load_dataset("valid", opt), fields, opt, is_train=False)

  # Do training.
  if len(opt.gpu_ranks):
    logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
  else:
    logger.info('Starting training on CPU, could be very slow')
  trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
                opt.valid_steps)

  if opt.tensorboard:
    trainer.report_manager.tensorboard_writer.close()
Example #6
0
def build_translator(opt, report_score=True, logger=None, out_file=None):
    if out_file is None:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')

    if opt.gpu > -1:
        torch.cuda.set_device(opt.gpu)

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    if len(opt.models) > 1:
        # use ensemble decoding if more than one model is specified
        fields, model, model_opt = \
            onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
    else:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(opt)

    translator = CodeTranslator(model,
                                fields,
                                opt,
                                model_opt,
                                global_scorer=scorer,
                                out_file=out_file,
                                report_score=report_score,
                                logger=logger)
    translator.src_path = opt.src
    return translator
Example #7
0
def build_translator(opt, report_score=True, logger=None, out_file=None):
    if out_file is None:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    if len(opt.models) > 1:
        # use ensemble decoding if more than one model is specified
        fields, model, model_opt = \
            onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
    else:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
                                             opt.beta,
                                             opt.coverage_penalty,
                                             opt.length_penalty)

    kwargs = {k: getattr(opt, k)
              for k in ["beam_size", "n_best", "max_length", "min_length",
                        "stepwise_penalty", "block_ngram_repeat",
                        "ignore_when_blocking", "dump_beam", "report_bleu",
                        "data_type", "replace_unk", "gpu", "verbose", "fast",
                        "sample_rate", "window_size", "window_stride",
                        "window", "image_channel_size"]}

    translator = Translator(model, fields, global_scorer=scorer,
                            out_file=out_file, report_score=report_score,
                            copy_attn=model_opt.copy_attn, logger=logger,
                            **kwargs)
    return translator
Example #8
0
def _get_parser():
    parser = ArgumentParser(description='train.py')

    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)
    return parser
Example #9
0
def _get_parser():
    parser = ArgumentParser(description='build_copy_transformer.py')

    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)
    return parser
Example #10
0
def main():
    parser = ArgumentParser()
    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.global_opts(parser)
    opt = parser.parse_args()
    with open(os.path.join(dir_path, 'opt_data'), 'wb') as f:
        pickle.dump(opt, f)
Example #11
0
def _get_parser():
    parser = ArgumentParser(description='RL_train.py')

    opts.config_opts(parser)
    # yida RL
    opts.model_opts(parser)
    # opts.train_opts(parser)
    opts.translate_opts(parser)
    return parser
Example #12
0
def build_translator(opt,
                     report_score=True,
                     out_file=None,
                     logger=None,
                     fields=None,
                     model=None,
                     model_opt=None):
    if out_file is None:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')
    # model should already be on device, is being used in part fo the training loop now
    # print(opt)
    # quit()
    # if opt.gpu > -1:
    #     torch.cuda.set_device(opt.gpu)

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    # if len(opt.models) > 1:
    #     # use ensemble decoding if more than one model is specified
    #     fields, model, model_opt = \
    #         onmt.decoders.ensemble.load_test_model(opt, dummy_opt.__dict__)
    # else:
    if fields is None and model is None and model_opt is None:
        fields, model, model_opt = \
            onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(
        0,  #opt.alpha,
        0,  #opt.beta,
        None,  #opt.coverage_penalty,
        None)  #opt.length_penalty)

    kwargs = {
        k: getattr(opt, k)
        for k in [
            "beam_size", "n_best", "max_length", "min_length",
            "stepwise_penalty", "block_ngram_repeat", "ignore_when_blocking",
            "dump_beam", "report_bleu", "data_type", "replace_unk", "verbose",
            "fast"
        ]
    }  # TODO gpu
    if opt.gpu_ranks:
        kwargs['gpu'] = True

    translator = Translator(model,
                            fields,
                            global_scorer=scorer,
                            out_file=out_file,
                            report_score=report_score,
                            copy_attn=model_opt.copy_attn,
                            logger=logger,
                            **kwargs)
    return translator
Example #13
0
def build_translator(opt):
    out_file = open(opt.output, 'w+', encoding='utf-8')

    dummy_parser = configargparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    load_test_model = onmt.decoders.ensemble.load_test_model \
        if len(opt.models) > 1 else onmt.model_builder.load_test_model
    fields, model, _ = load_test_model(opt, dummy_opt.__dict__)

    return Translator(model, fields, opt, out_file=out_file)
Example #14
0
def build_translator(opt,model_path):
  dummy_parser = configargparse.ArgumentParser(description='translate.py')
  opts.model_opts(dummy_parser)
  dummy_opt = dummy_parser.parse_known_args([])[0]
  #build the model and get the checkpoint and field
  path=[]
  path.append(model_path)
  opt.models=path
  fields, model = nmt_model.load_test_model(opt, dummy_opt.__dict__)

  translator = Translator(model, fields, opt)

  return translator
Example #15
0
def build_encoder_pass(opt, out_file=None):
    if out_file is None:
        out_file = codecs.open(opt.output, 'w+', 'utf-8')
    dummy_parser = configargparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    load_test_model = onmt.decoders.ensemble.load_test_model \
        if len(opt.models) > 1 else onmt.model_builder.load_test_model
    fields, model, model_opt = load_test_model(opt, dummy_opt.__dict__)

    encoder_pass = EncoderPass(model, fields, opt, out_file)
    return encoder_pass
Example #16
0
def parse_args():
    parser = configargparse.ArgumentParser(
        description='train.py',
        config_file_parser_class=configargparse.YAMLConfigFileParser,
        formatter_class=configargparse.ArgumentDefaultsHelpFormatter)

    opts.general_opts(parser)
    opts.config_opts(parser)
    opts.add_md_help_argument(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)

    opt = parser.parse_args()

    return opt
Example #17
0
def _get_parser():
    parser = ArgumentParser(description='train.py')
    parser.add_argument('--teacher_model_path',
                        action='store',
                        dest='teacher_model_path',
                        help='the path direct to the teacher model path')
    parser.add_argument("--word_sampling",
                        action="store",
                        default=False,
                        help="optional arg")

    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)
    return parser
Example #18
0
def _get_parser():
    parser = ArgumentParser(description='train.py')

    opts.config_opts(parser)
    opts.model_opts(parser)
    opts.train_opts(parser)
    '''extended opts for pretrained language models'''
    group = parser.add_argument_group("extended opts")
    group.add('--pretrained_encoder',
              '-pretrained_encoder',
              default="bert",
              type=str,
              choices=["bert", "roberta", "xlnet"],
              help="choose a pretrained language model as encoder")

    return parser
Example #19
0
def main():
    ## from translate.py
    parser = argparse.ArgumentParser(
        description='translate.py',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    onmt.opts.add_md_help_argument(parser)
    onmt.opts.translate_opts(parser)
    onmt.opts.model_opts(parser)
    logger.info("opts done")
    print("opts done")
    opt = parser.parse_args()

    logger.info("opt.gpu " + str(opt.gpu))
    print("opt.gpu " + str(opt.gpu))
    if opt.gpu > -1:
        torch.cuda.device(opt.gpu)
        n_gpu = 1
    else:
        n_gpu = 0

    # from train.py
    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    fields, model, model_opt = \
        onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    logger.info("bulding loss")
    print("building loss")
    valid_loss = onmt.utils.loss.build_loss_compute(
        model, fields["tgt"].vocab, opt, train=False)

    
    logger.info("Inference starts")
    print("Inference starts")
    infer = Inference(model, valid_loss, fields, trunc_size=0,
              shard_size=32, data_type='text', norm_method="sents",
              n_gpu=n_gpu, gpu_rank=1, gpu_verbose_level=0, report_manager=None)
    logger.info("translation")
    print("translation")

    infer.translate(src_path=opt.src,
                     tgt_path=opt.tgt,
                     src_dir=opt.src_dir,
                     batch_size=opt.batch_size,
                     ans_path=opt.ans)
Example #20
0
def build_translator(opt):
    dummy_parser = argparse.ArgumentParser(description="train.py")
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    if len(opt.models) > 1:
        # use ensemble decoding if more than one model is specified
        fields, model, model_opt = onmt.decoders.ensemble.load_test_model(
            opt, dummy_opt.__dict__)
    else:
        fields, model, model_opt = onmt.model_builder.load_test_model(
            opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(opt.alpha, opt.beta,
                                             opt.coverage_penalty,
                                             opt.length_penalty)

    kwargs = {
        k: getattr(opt, k)
        for k in [
            "beam_size",
            "n_best",
            "max_length",
            "min_length",
            "stepwise_penalty",
            "block_ngram_repeat",
            "ignore_when_blocking",
            "data_type",
            "replace_unk",
            "gpu",
            "fast",
            "sample_rate",
            "window_size",
            "window_stride",
            "window",
            "image_channel_size",
            "mask_from",
        ]
    }

    translator = Translator(model,
                            fields,
                            global_scorer=scorer,
                            copy_attn=model_opt.copy_attn,
                            **kwargs)
    return translator
            'opt': self.model_opt,
            'optim': self.optim,
        }

        logger.info("Saving checkpoint %s_step_%d.pt" % (self.base_path, step))
        checkpoint_path = '%s_step_%d.pt' % (self.base_path, step)
        torch.save(checkpoint, checkpoint_path)
        return checkpoint, checkpoint_path

    def _rm_checkpoint(self, name):
        """
        Remove a checkpoint

        Args:
            name(str): name that indentifies the checkpoint
                (it may be a filepath)
        """
        os.remove(name)


if __name__ == "__main__":
    parser = configargparse.ArgumentParser(
        description='train.py',
        formatter_class=configargparse.ArgumentDefaultsHelpFormatter)

    opts.model_opts(parser)
    opts.train_opts(parser)

    opt = parser.parse_args()
    main(opt)
Example #22
0
def main(opt, device_id):
    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    out_file = None
    best_test_score, best_ckpt = -10000, None
    dummy_parser = argparse.ArgumentParser(description='meta_dev_zs.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    for i in range(57, opt.meta_iterations * opt.inner_iterations + 57, 57 * 5):
        ckpt_path = '{}_epoch_{}.pt'.format(opt.save_model, i)
        logger.info('Loading checkpoint from %s' % ckpt_path)
        
        if out_file is None:
            out_file = codecs.open(opt.output, 'w+', 'utf-8')
        fields, model, model_opt = onmt.model_builder.load_test_model(opt, dummy_opt.__dict__, model_path=ckpt_path)

        scorer = onmt.translate.GNMTGlobalScorer(opt.alpha,
                                                opt.beta,
                                                opt.coverage_penalty,
                                                opt.length_penalty)

        kwargs = {k: getattr(opt, k)
                    for k in ["beam_size", "n_best", "max_length", "min_length",
                                "stepwise_penalty", "block_ngram_repeat",
                                "ignore_when_blocking", "dump_beam", "report_bleu",
                                "replace_unk", "gpu", "verbose", "fast", "mask_from"]}
        fields['graph'] = torchtext.data.Field(sequential = False)
        translator = Translator(model, fields, global_scorer=scorer,
                                out_file=out_file, report_score=False,
                                copy_attn=model_opt.copy_attn, logger=logger,
                                log_probs_out_file=None,
                                **kwargs)
        # make translation and save result
        all_scores, all_predictions = translator.translate(
            src_path='processed_data/meta-dev/' + opt.meta_dev_task + '/src-test.txt',
            tgt_path=None,
            src_dir=None,
            batch_size=opt.translate_batch_size,
            attn_debug=False)
        # dump predictions
        f = open('experiments/meta_dev/' + opt.meta_dev_task + '/test_predictions.csv', 'w', encoding='utf-8')
        f.write('smiles,property\n')
        for n_best_mols in all_predictions:
            for mol in n_best_mols:
                f.write(mol.replace(' ', '')+',0\n')
        f.close()
        # call chemprop to get scores
        test_path = '\"' + 'experiments/meta_dev/' + opt.meta_dev_task + '/test_predictions.csv' + '\"'
        checkpoint_path = '\"' + 'scorer_ckpts/' + opt.meta_dev_task + '/model.pt' + '\"'
        preds_path = '\"' + 'experiments/meta_dev/' + opt.meta_dev_task + '/test_scores.csv' + '\"'
        
        # in case of all mols are invalid (will produce not output file by chemprop)
        # the predictions are copied into score file
        cmd = 'cp {} {}'.format(test_path, preds_path)
        result = os.popen(cmd)
        result.close()

        cmd = 'python chemprop/predict.py --test_path {} --checkpoint_path {} --preds_path {} --num_workers 0'.format(test_path, checkpoint_path, preds_path)
        scorer_result = os.popen(cmd)
        # logger.info('{}'.format('\n'.join(scorer_result.readlines())))
        scorer_result.close()
        # read score file and get score

        score = read_score_csv('experiments/meta_dev/' + opt.meta_dev_task + '/test_scores.csv')
        
        assert len(score) % opt.beam_size == 0

        # report if it is the best on test
        test_metrics = calculate_metrics(opt.meta_dev_task, 'dev', 'test', score)
        logger.info('test metrics: ' + str(test_metrics))
        test_score = test_metrics['success_rate']
        if test_score > best_test_score:
            best_ckpt = ckpt_path
            logger.info('New best test success rate: {:.4f} by {}'.format(test_score, ckpt_path))
            best_test_score = test_score
        else:
            logger.info('test success rate: {:.4f} by {}'.format(test_score, ckpt_path))

        del model_opt
        del fields
        del model
        gc.collect()
Example #23
0
import onmt.opts as opts
from train_multi import main as multi_main
from train_single import main as single_main


def main(opt):

    if opt.rnn_type == "SRU" and not opt.gpuid:
        raise AssertionError("Using SRU requires -gpuid set.")

    if torch.cuda.is_available() and not opt.gpuid:
        print("WARNING: You have a CUDA device, should run with -gpuid 0")

    if len(opt.gpuid) > 1:
        multi_main(opt)
    else:
        single_main(opt)


if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(
        description='train.py',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    opts.add_md_help_argument(PARSER)
    opts.model_opts(PARSER)
    opts.train_opts(PARSER)

    OPT = PARSER.parse_args()
    main(OPT)
Example #24
0
def main(opt, device_id):
    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        logger.info('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)

        # Load default opts values then overwrite it with opts from
        # the checkpoint. It's usefull in order to re-train a model
        # after adding a new option (not set in checkpoint)
        dummy_parser = configargparse.ArgumentParser()
        opts.model_opts(dummy_parser)
        default_opt = dummy_parser.parse_known_args([])[0]

        model_opt = default_opt
        model_opt.__dict__.update(checkpoint['opt'].__dict__)
        logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
        vocab = checkpoint['vocab']
    else:
        checkpoint = None
        model_opt = opt
        vocab = torch.load(opt.data + '.vocab.pt')

    # check for code where vocab is saved instead of fields
    # (in the future this will be done in a smarter way)
    if old_style_vocab(vocab):
        data_type = opt.model_type
        fields = load_old_vocab(vocab, data_type, dynamic_dict=opt.copy_attn)
    else:
        fields = vocab

    # Report src and tgt vocab sizes, including for features
    for side in ['src', 'tgt']:
        for name, f in fields[side]:
            try:
                f_iter = iter(f)
            except TypeError:
                f_iter = [(name, f)]
            for sn, sf in f_iter:
                if sf.use_vocab:
                    logger.info(' * %s vocab size = %d' % (sn, len(sf.vocab)))

    # Build model.
    model = build_model(model_opt, opt, fields, checkpoint)
    n_params, enc, dec = _tally_parameters(model)
    logger.info('encoder: %d' % enc)
    logger.info('decoder: %d' % dec)
    logger.info('* number of parameters: %d' % n_params)
    _check_save_model_path(opt)

    # Build optimizer.
    optim = Optimizer.from_opt(model, opt, checkpoint=checkpoint)

    # Build model saver
    model_saver = build_model_saver(model_opt, opt, model, fields, optim)

    trainer = build_trainer(opt,
                            device_id,
                            model,
                            fields,
                            optim,
                            model_saver=model_saver)

    # this line is kind of a temporary kludge because different objects expect
    # fields to have a different structure
    dataset_fields = dict(chain.from_iterable(fields.values()))

    train_iter = build_dataset_iter("train", dataset_fields, opt)
    valid_iter = build_dataset_iter("valid",
                                    dataset_fields,
                                    opt,
                                    is_train=False)

    if len(opt.gpu_ranks):
        logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
    else:
        logger.info('Starting training on CPU, could be very slow')
    train_steps = opt.train_steps
    if opt.single_pass and train_steps > 0:
        logger.warning("Option single_pass is enabled, ignoring train_steps.")
        train_steps = 0
    trainer.train(train_iter,
                  train_steps,
                  save_checkpoint_steps=opt.save_checkpoint_steps,
                  valid_iter=valid_iter,
                  valid_steps=opt.valid_steps)

    if opt.tensorboard:
        trainer.report_manager.tensorboard_writer.close()
def build_translator(
        model='forward_models/fine_tuned_model_on_liu_dataset_0p02.pt',
        use_gpu=1,
        batch_size=500,
        report_score=False,
        logger=None,
        out_file=None,
        log_probs_out_file=None):

    opt_dict = {
        'alpha': 0.0,
        'beam_size': 5,
        'beta': -0.0,
        'block_ngram_repeat': 0,
        'coverage_penalty': 'none',
        'data_type': 'text',
        'dump_beam': '',
        'dynamic_dict': False,
        'fast': True,
        'gpu': use_gpu,
        'ignore_when_blocking': [],
        'image_channel_size': 3,
        'length_penalty': 'none',
        'log_file': '',
        'log_probs': True,
        'mask_from': '',
        'max_length': 200,
        'max_sent_length': None,
        'min_length': 0,
        'models': [model],
        'n_best': 5,
        'replace_unk': True,
        'report_bleu': False,
        'report_rouge': False,
        'sample_rate': 16000,
        'share_vocab': False,
        'stepwise_penalty': False,
        'tgt': None,
        'verbose': False,
        'window': 'hamming',
        'window_size': 0.02,
        'window_stride': 0.01
    }

    opt = argparse.Namespace()
    for name in opt_dict.keys():
        setattr(opt, name, opt_dict[name])

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    fields, model, model_opt = \
        onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    scorer = onmt.translate.GNMTGlobalScorer(opt.alpha, opt.beta,
                                             opt.coverage_penalty,
                                             opt.length_penalty)

    kwargs = {
        k: getattr(opt, k)
        for k in [
            "beam_size", "n_best", "max_length", "min_length",
            "stepwise_penalty", "block_ngram_repeat", "ignore_when_blocking",
            "dump_beam", "report_bleu", "data_type", "replace_unk", "gpu",
            "verbose", "fast", "sample_rate", "window_size", "window_stride",
            "window", "image_channel_size", "mask_from"
        ]
    }

    translator = Translator(model,
                            fields,
                            global_scorer=scorer,
                            out_file=out_file,
                            report_score=report_score,
                            copy_attn=model_opt.copy_attn,
                            logger=logger,
                            log_probs_out_file=log_probs_out_file,
                            **kwargs)
    return translator
Example #26
0
def main(opt, device_id):
    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        logger.info('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)

        # Load default opts values then overwrite it with opts from
        # the checkpoint. It's usefull in order to re-train a model
        # after adding a new option (not set in checkpoint)
        dummy_parser = configargparse.ArgumentParser()
        opts.model_opts(dummy_parser)
        default_opt = dummy_parser.parse_known_args([])[0]

        model_opt = default_opt
        model_opt.__dict__.update(checkpoint['opt'].__dict__)
    else:
        checkpoint = None
        model_opt = opt

    # Peek the first dataset to determine the data_type.
    # (All datasets have the same data_type).
    first_dataset = next(lazily_load_dataset("train", opt))
    # print("[onmt.train_single.py] first_dataset.examples[0]: {}".format(first_dataset.examples[0]))
    # print("[onmt.train_single.py] first_dataset.examples[0].src[:10]: {}".format(first_dataset.examples[0].src[:10]))
    print("[onmt.train_single.py] first_dataset.examples[0].src_da_label: {}".
          format(first_dataset.examples[0].src_da_label))
    print(
        "[onmt.train_single.py] first_dataset.examples[0].__dict__.keys(): {}".
        format(first_dataset.examples[0].__dict__.keys()))
    data_type = first_dataset.data_type
    print("[onmt.train_single.py] first_dataset.data_type: {}".format(
        first_dataset.data_type))

    # Load fields generated from preprocess phase.
    fields = load_fields(first_dataset, opt, checkpoint)

    # Report src/tgt features.

    knl_features, src_features, tgt_features = _collect_report_features(fields)
    for j, feat in enumerate(src_features):
        logger.info(' * knl feature %d size = %d' %
                    (j, len(fields[feat].vocab)))
    for j, feat in enumerate(src_features):
        logger.info(' * src feature %d size = %d' %
                    (j, len(fields[feat].vocab)))
    for j, feat in enumerate(tgt_features):
        logger.info(' * tgt feature %d size = %d' %
                    (j, len(fields[feat].vocab)))

    # Build model.
    model = build_model(model_opt, opt, fields, checkpoint)
    n_params, enc, dec = _tally_parameters(model)
    logger.info('encoder: %d' % enc)
    logger.info('decoder: %d' % dec)
    logger.info('* number of parameters: %d' % n_params)
    _check_save_model_path(opt)

    # Build optimizer.
    optim = build_optim(model, opt, checkpoint)

    # Build model saver
    model_saver = build_model_saver(model_opt, opt, model, fields, optim)

    trainer = build_trainer(opt,
                            device_id,
                            model,
                            fields,
                            optim,
                            data_type,
                            model_saver=model_saver)

    def train_iter_fct():
        return build_dataset_iter(lazily_load_dataset("train", opt), fields,
                                  opt)

    def valid_iter_fct():
        return build_dataset_iter(lazily_load_dataset("valid", opt),
                                  fields,
                                  opt,
                                  is_train=False)

    # Do training.
    if len(opt.gpu_ranks):
        logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
    else:
        logger.info('Starting training on CPU, could be very slow')
    trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
                  opt.valid_steps)

    if opt.tensorboard:
        trainer.report_manager.tensorboard_writer.close()
Example #27
0
def main(opt, device_id):
    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        logger.info('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)

        # Load default opts values then overwrite it with opts from
        # the checkpoint. It's usefull in order to re-train a model
        # after adding a new option (not set in checkpoint)
        dummy_parser = configargparse.ArgumentParser()
        opts.model_opts(dummy_parser)
        default_opt = dummy_parser.parse_known_args([])[0]

        model_opt = default_opt
        model_opt.__dict__.update(checkpoint['opt'].__dict__)
        logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
        vocab = checkpoint['vocab']
    else:
        checkpoint = None
        model_opt = opt
        vocab = torch.load(opt.data + '.vocab.pt')

    # Load a shard dataset to determine the data_type.
    # (All datasets have the same data_type).
    # this should be refactored out of existence reasonably soon
    first_dataset = torch.load(glob.glob(opt.data + '.train*.pt')[0])
    data_type = first_dataset.data_type

    # check for code where vocab is saved instead of fields
    # (in the future this will be done in a smarter way
    if old_style_vocab(vocab):
        fields = load_fields_from_vocab(vocab, data_type)
    else:
        fields = vocab

    # Report src and tgt vocab sizes, including for features
    for side in ['src', 'tgt']:
        for name, f in fields[side]:
            if f.use_vocab:
                logger.info(' * %s vocab size = %d' % (name, len(f.vocab)))

    # Build model.
    model = build_model(model_opt, opt, fields, checkpoint)
    n_params, enc, dec = _tally_parameters(model)
    logger.info('encoder: %d' % enc)
    logger.info('decoder: %d' % dec)
    logger.info('* number of parameters: %d' % n_params)
    _check_save_model_path(opt)

    # Build optimizer.
    optim = build_optim(model, opt, checkpoint)

    # Build model saver
    model_saver = build_model_saver(model_opt, opt, model, fields, optim)

    trainer = build_trainer(opt,
                            device_id,
                            model,
                            fields,
                            optim,
                            data_type,
                            model_saver=model_saver)

    # this line is kind of a temporary kludge because different objects expect
    # fields to have a different structure
    dataset_fields = dict(chain.from_iterable(fields.values()))

    train_iter = build_dataset_iter("train", dataset_fields, opt)
    valid_iter = build_dataset_iter("valid",
                                    dataset_fields,
                                    opt,
                                    is_train=False)

    if len(opt.gpu_ranks):
        logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
    else:
        logger.info('Starting training on CPU, could be very slow')
    trainer.train(train_iter, valid_iter, opt.train_steps, opt.valid_steps)

    if opt.tensorboard:
        trainer.report_manager.tensorboard_writer.close()
Example #28
0
def main(opt):
    if opt.gpuid:
        raise AssertionError("gpuid is deprecated \
              see world_size and gpu_ranks")

    assert opt.world_size <= 1, "you don't need multi-gpu for morphology"

    device_id = 0 if len(opt.gpu_ranks) == 1 else -1

    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        logger.info('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)

        # Load default opts values then overwrite it with opts from
        # the checkpoint. It's useful in order to re-train a model
        # after adding a new option (not set in checkpoint)
        dummy_parser = configargparse.ArgumentParser()
        opts.model_opts(dummy_parser)
        default_opt = dummy_parser.parse_known_args([])[0]

        model_opt = default_opt
        model_opt.__dict__.update(checkpoint['opt'].__dict__)
        logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
        fields = checkpoint['vocab']
    else:
        checkpoint = None
        model_opt = opt
        fields = torch.load(opt.data + '.vocab.pt')

    for key, values in fields.items():
        for name, f in values:
            if hasattr(f, 'use_vocab') and f.use_vocab:
                logger.info(' * %s vocab size = %d' % (name, len(f.vocab)))

    # Build model.
    logger.info('Building model...')
    model = build_model(model_opt, fields, use_gpu(opt), checkpoint)
    logger.info(model)
    n_params, enc, dec = _tally_parameters(model)
    logger.info('encoder: %d' % enc)
    logger.info('decoder: %d' % dec)
    logger.info('* number of parameters: %d' % n_params)
    _check_save_model_path(opt)

    # Build optimizer.
    params = model.parameters()
    optim_args = {"lr": opt.learning_rate}
    if opt.optim == "adam":
        # no need to mess with the default betas
        optim_args["eps"] = 1e-9
    elif opt.optim == "adagrad":
        optim_args["initial_accumulator_value"] = opt.adagrad_accumulator_init
    optim = getattr(torch.optim, opt.optim.title())(params, **optim_args)
    print(optim)

    trainer = build_trainer(opt, model_opt, device_id, model, fields, optim)

    # this line is kind of a temporary kludge because different objects expect
    # fields to have a different structure
    dataset_fields = dict(chain.from_iterable(fields.values()))

    device = "cuda" if opt.gpu_ranks else "cpu"

    train_dataset = torch.load(opt.data + '.train.pt')
    train_dataset.fields = dataset_fields
    train_iter = OrderedIterator(
        train_dataset, opt.batch_size, sort_within_batch=True,
        device=device, repeat=False, shuffle=not opt.no_shuffle)

    valid_dataset = torch.load(opt.data + '.valid.pt')
    valid_dataset.fields = dataset_fields
    valid_iter = OrderedIterator(
        valid_dataset, opt.valid_batch_size, train=False,
        sort_within_batch=True, device=device)

    logger.info('Starting training on {}'.format(device))
    trainer.train(train_iter, valid_iter, opt.epochs)
Example #29
0
def main(opt, device_id):
    opt = training_opt_postprocessing(opt, device_id)
    init_logger(opt.log_file)
    # Gather information related to the training script and commit version
    script_path = os.path.abspath(__file__)
    script_dir = os.path.dirname(os.path.dirname(script_path))
    logger.info('Train script dir: %s' % script_dir)
    git_commit = str(subprocess.check_output(['bash', script_dir + '/cluster_scripts/git_version.sh']))
    logger.info("Git Commit: %s" % git_commit[2:-3])
    # Load checkpoint if we resume from a previous training.
    if opt.train_from:
        # TODO: load MTL model
        logger.info('Loading checkpoint from %s' % opt.train_from)
        checkpoint = torch.load(opt.train_from,
                                map_location=lambda storage, loc: storage)

        # Load default opts values then overwrite it with opts from
        # the checkpoint. It's usefull in order to re-train a model
        # after adding a new option (not set in checkpoint)
        dummy_parser = configargparse.ArgumentParser()
        opts.model_opts(dummy_parser)
        default_opt = dummy_parser.parse_known_args([])[0]

        model_opt = default_opt
        model_opt.__dict__.update(checkpoint['opt'].__dict__)
    else:
        checkpoint = None
        model_opt = opt


    num_tasks = len(opt.data.split(','))
    opt.num_tasks = num_tasks

    checkpoint_list=[]
    if opt.warm_model:
        base_name=opt.warm_model
        for task_id in range(num_tasks):
            chkpt_path=base_name.replace("X",str(task_id))
            if not os.path.isfile(chkpt_path):
                chkpt_path = base_name.replace("X", str(0))
            logger.info('Loading a checkpoint from %s' % chkpt_path)

            checkpoint = torch.load(chkpt_path,
                                    map_location=lambda storage, loc: storage)
            checkpoint_list.append(checkpoint)
    else:
        for task_id in range(num_tasks):
            checkpoint_list.append(None)

    fields_list = []
    data_type=None
    for task_id in range(num_tasks):
        # Peek the first dataset to determine the data_type.
        # (All datasets have the same data_type).
        first_dataset = next(lazily_load_dataset("train", opt, task_id=task_id))
        data_type = first_dataset.data_type

        # Load fields generated from preprocess phase.
        if opt.mtl_shared_vocab and task_id > 0:
            logger.info(' * vocabulary size. Same as the main task!')
            fields = fields_list[0]
        else:
            fields = load_fields(first_dataset, opt, checkpoint_list[task_id], task_id=task_id)

        # Report src/tgt features.

        src_features, tgt_features = _collect_report_features(fields)
        for j, feat in enumerate(src_features):
            logger.info(' * (Task %d) src feature %d size = %d'
                        % (task_id, j, len(fields[feat].vocab)))
        for j, feat in enumerate(tgt_features):
            logger.info(' * (Task %) tgt feature %d size = %d'
                        % (task_id, j, len(fields[feat].vocab)))
        fields_list.append(fields)

    if opt.epochs > -1:
        total_num_batch = 0
        for task_id in range(num_tasks):
            train_iter = build_dataset_iter(lazily_load_dataset("train", opt, task_id=task_id), fields_list[task_id], opt)
            for i, batch in enumerate(train_iter):
                num_batch = i
            total_num_batch+=num_batch
            if opt.mtl_schedule < 10:
                break
        num_batch = total_num_batch
        opt.train_steps = (num_batch * opt.epochs) + 1
        # Do the validation and save after each epoch
        opt.valid_steps = num_batch
        opt.save_checkpoint_steps = 1

    # logger.info(opt_to_string(opt))
    logger.info(opt)

    # Build model(s).
    models_list = []
    for task_id in range(num_tasks):
        if opt.mtl_fully_share and task_id > 0:
            # Since we only have one model, copy the pointer to the model for all
            models_list.append(models_list[0])
        else:

            main_model = models_list[0] if task_id > 0 else None
            model = build_model(model_opt, opt, fields_list[task_id], checkpoint_list[task_id], main_model=main_model, task_id=task_id)
            n_params, enc, dec = _tally_parameters(model)
            logger.info('(Task %d) encoder: %d' % (task_id, enc))
            logger.info('(Task %d) decoder: %d' % (task_id, dec))
            logger.info('* number of parameters: %d' % n_params)
            _check_save_model_path(opt)
            models_list.append(model)

    # combine parameters of different models and consider shared parameters just once.
    def combine_named_parameters(named_params_list):
        observed_params = []
        for model_named_params in named_params_list:
            for name, p in model_named_params:
                is_observed = False
                # Check whether we observed this parameter before
                for param in observed_params:
                    if p is param:
                        is_observed = True
                        break
                if not is_observed:
                    observed_params.append(p)
                    yield name, p

    # Build optimizer.
    optims_list = []
    all_models_params=[]
    for task_id in range(num_tasks):
        if not opt.mtl_shared_optimizer:
            optim = build_optim(models_list[task_id], opt, checkpoint)
            optims_list.append(optim)
        else:
            all_models_params.append(models_list[task_id].named_parameters())

    # Extract the list of shared parameters among the models of all tasks.
    observed_params = []
    shared_params = []
    for task_id in range(num_tasks):
        for name, p in models_list[task_id].named_parameters():
            is_observed = False
            # Check whether we observed this parameter before
            for param in observed_params:
                if p is param:
                    shared_params.append(name)
                    is_observed = True
                    break
            if not is_observed:
                observed_params.append(p)
    opt.shared_params = shared_params

    if opt.mtl_shared_optimizer:
        optim = build_optim_mtl_params(combine_named_parameters(all_models_params), opt, checkpoint)
        optims_list.append(optim)

    # Build model saver
    model_saver = build_mtl_model_saver(model_opt, opt, models_list, fields_list, optims_list)

    trainer = build_trainer(opt, device_id, models_list, fields_list,
                            optims_list, data_type, model_saver=model_saver)

    def train_iter_fct(task_id):
        return build_dataset_iter(
            lazily_load_dataset("train", opt, task_id=task_id), fields_list[task_id], opt)

    def valid_iter_fct(task_id):
        return build_dataset_iter(
            lazily_load_dataset("valid", opt, task_id=task_id), fields_list[task_id], opt)

    def meta_valid_iter_fct(task_id, is_log=False):
        return build_dataset_iter(
            lazily_load_dataset("meta_valid", opt, task_id=task_id, is_log=is_log), fields_list[task_id], opt)

    # Do training.
    if len(opt.gpu_ranks):
        logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
    else:
        logger.info('Starting training on CPU, could be very slow')
    trainer.train(train_iter_fct, valid_iter_fct, opt.train_steps,
                  opt.valid_steps, meta_valid_iter_fct=meta_valid_iter_fct)

    if opt.tensorboard:
        trainer.report_manager.tensorboard_writer.close()
Example #30
0
def translate(opt):
    out_file = codecs.open(opt.output, 'w+', 'utf-8')

    if opt.gpu > -1:
        torch.cuda.set_device(opt.gpu)

    dummy_parser = argparse.ArgumentParser(description='train.py')
    opts.model_opts(dummy_parser)
    dummy_opt = dummy_parser.parse_known_args([])[0]

    fields, model, model_opt = \
        onmt.model_builder.load_test_model(opt, dummy_opt.__dict__)

    data = inputters.build_dataset(fields,
                                   'text',
                                   src_path=opt.src,
                                   src_data_iter=None,
                                   tgt_path=opt.tgt,
                                   tgt_data_iter=None,
                                   src_dir=opt.src_dir,
                                   sample_rate='16000',
                                   window_size=.02,
                                   window_stride=.01,
                                   window='hamming',
                                   use_filter_pred=False)

    device = torch.device('cuda' if opt.gpu > -1 else 'cpu')

    batch_size = 1

    data_iter = inputters.OrderedIterator(dataset=data,
                                          device=device,
                                          batch_size=batch_size,
                                          train=False,
                                          sort=False,
                                          sort_within_batch=True,
                                          shuffle=False)

    pair_size = model_opt.wpe_pair_size

    s_id = fields["tgt"].vocab.stoi['<s>']
    if '<sgo>' in fields["tgt"].vocab.stoi:
        ss_id = fields["tgt"].vocab.stoi['<sgo>']
    else:
        ss_id = fields['tgt'].vocab.stoi['<unk>']
    if '<seos>' in fields['tgt'].vocab.stoi:
        eos_id = fields['tgt'].vocab.stoi['<seos>']
    else:
        eos_id = fields['tgt'].vocab.stoi['</s>']

    for i, batch in enumerate(data_iter):
        tgt = torch.LongTensor([s_id] * batch_size + [ss_id] *
                               ((pair_size - 1) * batch_size)).view(
                                   pair_size,
                                   batch_size).unsqueeze(2).to(device)
        dec_state = None
        src = inputters.make_features(batch, 'src', 'text')
        _, src_lengths = batch.src

        result = None

        for _ in range(opt.max_length):
            outputs, _, dec_state = model(src, tgt, src_lengths, dec_state)
            scores = model.generator(outputs.view(-1, outputs.size(2)))
            indices = scores.argmax(dim=1)
            tgt = indices.view(pair_size, batch_size,
                               1)  # (pair_size x batch x feat)

            assert batch_size == 1
            if tgt[0][0][0].item() == eos_id:
                break

            if result is None:
                result = indices.view(pair_size, batch_size)
            else:
                result = torch.cat(
                    [result, indices.view(pair_size, batch_size)], 0)

        result = result.transpose(0, 1).tolist()
        for sent in result:
            sent = [fields["tgt"].vocab.itos[_] for _ in sent]
            sent = [_ for _ in sent if _ not in ['<blank>', '<seos>', '</s>']]
            sent = ' '.join(sent)
            out_file.write(sent + '\n')

        print('Translated {} batches'.format(i))

    out_file.close()