示例#1
0
        '--resume_epoch',
        type=int,
        default=None,
        help=
        'resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)'
    )
    args = parser.parse_args()
    args.max_seq_len = WORD_MAX_SEQ_LEN
    args.sort_k_batches = WORD_SORT_K_BATCHES

    logger, opts = setup_exp(args)

    logger.info('loading data...')
    train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE,
                                                 TGT_TRAIN_FILE,
                                                 args.batch_size,
                                                 WORD_SORT_K_BATCHES,
                                                 training=True)
    logger.info('done loading data')
    src_bdict = {v: k for k, v in src_dict.iteritems()}
    tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
    valid_stream, _, _ = load_data(SRC_VALID_FILE,
                                   TGT_VALID_FILE,
                                   args.batch_size,
                                   WORD_SORT_K_BATCHES,
                                   training=False)

    if args.resume_epoch:
        assert (args.resume_epoch > 0)
        start_epoch = args.resume_epoch
        logger.info('loading model...')
示例#2
0
                        help='insertion bonus')
    args = parser.parse_args()
    args.reverse = True  # just in case

    logger = get_logger(args.expdir)

    def model_fn(x):
        return EncoderDecoder(x)

    model, opts = restore_model_and_opts(model_fn, args, logger)

    # TODO try w/ test as well
    if not args.test:
        eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE,
                                                    TGT_VALID_FILE,
                                                    opts['batch_size'],
                                                    WORD_SORT_K_BATCHES,
                                                    training=False)
    else:
        eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE,
                                                    TGT_TEST_FILE,
                                                    opts['batch_size'],
                                                    WORD_SORT_K_BATCHES,
                                                    training=False)
    tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}

    # finally, run decoding

    decoded_sentences = list()

    decode_costs = []
示例#3
0
    parser.add_argument('--dropout', type=float, default=0.0, help='dropout (fraction of units randomly dropped on non-recurrent connections)')
    parser.add_argument('--src_vocab_size', type=int, default=SRC_VOCAB_SIZE, help='source vocabulary size')
    parser.add_argument('--tgt_vocab_size', type=int, default=TGT_VOCAB_SIZE, help='source vocabulary size')
    parser.add_argument('--epochs', type=int, default=8, help='number of epochs to train')
    parser.add_argument('--print_every', type=int, default=1, help='how often to print cost')
    parser.add_argument('--optimizer', type=str, default='sgd', choices=optimizers)
    parser.add_argument('--reverse', action='store_true', help='reverse source input sentence')
    parser.add_argument('--expdir', type=str, default='sandbox', help='experiment directory to save files to')
    parser.add_argument('--resume_epoch', type=int, default=None, help='resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)')
    args = parser.parse_args()
    args.max_seq_len = WORD_MAX_SEQ_LEN; args.sort_k_batches = WORD_SORT_K_BATCHES

    logger, opts = setup_exp(args)

    logger.info('loading data...')
    train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE, TGT_TRAIN_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=True)
    logger.info('done loading data')
    src_bdict = {v: k for k, v in src_dict.iteritems()}
    tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
    valid_stream, _, _ = load_data(SRC_VALID_FILE, TGT_VALID_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=False)

    if args.resume_epoch:
        assert (args.resume_epoch > 0)
        start_epoch = args.resume_epoch
        logger.info('loading model...')
        encdec = EncoderDecoder(args)
        load_model_params(encdec, pjoin(args.expdir, 'model_epoch%d.h5' % (args.resume_epoch - 1)))
        logger.info('done loading model')
    else:
        start_epoch = 0
        encdec = EncoderDecoder(args)
示例#4
0
    parser.add_argument('expdir', type=str, help='experiment directory')
    parser.add_argument('model', type=str, help='model file to use')
    parser.add_argument('outfile', type=str, help='file to write decoder output setnences to')
    parser.add_argument('--beam', type=int, default=12, help='beam size to use')
    parser.add_argument('--test', action='store_true', help='evaluate on test set instead of validation set')
    parser.add_argument('--beta', type=float, default=1.0, help='insertion bonus')
    args = parser.parse_args()
    args.reverse = True  # just in case

    logger = get_logger(args.expdir)
    def model_fn(x): return EncoderDecoder(x)
    model, opts = restore_model_and_opts(model_fn, args, logger)

    # TODO try w/ test as well
    if not args.test:
        eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE, TGT_VALID_FILE,
            opts['batch_size'], WORD_SORT_K_BATCHES, training=False)
    else:
        eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE, TGT_TEST_FILE,
            opts['batch_size'], WORD_SORT_K_BATCHES, training=False)
    tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}

    # finally, run decoding

    decoded_sentences = list()

    decode_costs = []
    lengths = []
    logger.info('starting decoding')
    tic = time.time()
    sent_ind = 0
    for ss, sm, ts, tm in eval_stream.get_epoch_iterator():