'--resume_epoch', type=int, default=None, help= 'resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)' ) args = parser.parse_args() args.max_seq_len = WORD_MAX_SEQ_LEN args.sort_k_batches = WORD_SORT_K_BATCHES logger, opts = setup_exp(args) logger.info('loading data...') train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE, TGT_TRAIN_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=True) logger.info('done loading data') src_bdict = {v: k for k, v in src_dict.iteritems()} tgt_bdict = {v: k for k, v in tgt_dict.iteritems()} valid_stream, _, _ = load_data(SRC_VALID_FILE, TGT_VALID_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=False) if args.resume_epoch: assert (args.resume_epoch > 0) start_epoch = args.resume_epoch logger.info('loading model...')
help='insertion bonus') args = parser.parse_args() args.reverse = True # just in case logger = get_logger(args.expdir) def model_fn(x): return EncoderDecoder(x) model, opts = restore_model_and_opts(model_fn, args, logger) # TODO try w/ test as well if not args.test: eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE, TGT_VALID_FILE, opts['batch_size'], WORD_SORT_K_BATCHES, training=False) else: eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE, TGT_TEST_FILE, opts['batch_size'], WORD_SORT_K_BATCHES, training=False) tgt_bdict = {v: k for k, v in tgt_dict.iteritems()} # finally, run decoding decoded_sentences = list() decode_costs = []
parser.add_argument('--dropout', type=float, default=0.0, help='dropout (fraction of units randomly dropped on non-recurrent connections)') parser.add_argument('--src_vocab_size', type=int, default=SRC_VOCAB_SIZE, help='source vocabulary size') parser.add_argument('--tgt_vocab_size', type=int, default=TGT_VOCAB_SIZE, help='source vocabulary size') parser.add_argument('--epochs', type=int, default=8, help='number of epochs to train') parser.add_argument('--print_every', type=int, default=1, help='how often to print cost') parser.add_argument('--optimizer', type=str, default='sgd', choices=optimizers) parser.add_argument('--reverse', action='store_true', help='reverse source input sentence') parser.add_argument('--expdir', type=str, default='sandbox', help='experiment directory to save files to') parser.add_argument('--resume_epoch', type=int, default=None, help='resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)') args = parser.parse_args() args.max_seq_len = WORD_MAX_SEQ_LEN; args.sort_k_batches = WORD_SORT_K_BATCHES logger, opts = setup_exp(args) logger.info('loading data...') train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE, TGT_TRAIN_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=True) logger.info('done loading data') src_bdict = {v: k for k, v in src_dict.iteritems()} tgt_bdict = {v: k for k, v in tgt_dict.iteritems()} valid_stream, _, _ = load_data(SRC_VALID_FILE, TGT_VALID_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=False) if args.resume_epoch: assert (args.resume_epoch > 0) start_epoch = args.resume_epoch logger.info('loading model...') encdec = EncoderDecoder(args) load_model_params(encdec, pjoin(args.expdir, 'model_epoch%d.h5' % (args.resume_epoch - 1))) logger.info('done loading model') else: start_epoch = 0 encdec = EncoderDecoder(args)
parser.add_argument('expdir', type=str, help='experiment directory') parser.add_argument('model', type=str, help='model file to use') parser.add_argument('outfile', type=str, help='file to write decoder output setnences to') parser.add_argument('--beam', type=int, default=12, help='beam size to use') parser.add_argument('--test', action='store_true', help='evaluate on test set instead of validation set') parser.add_argument('--beta', type=float, default=1.0, help='insertion bonus') args = parser.parse_args() args.reverse = True # just in case logger = get_logger(args.expdir) def model_fn(x): return EncoderDecoder(x) model, opts = restore_model_and_opts(model_fn, args, logger) # TODO try w/ test as well if not args.test: eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE, TGT_VALID_FILE, opts['batch_size'], WORD_SORT_K_BATCHES, training=False) else: eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE, TGT_TEST_FILE, opts['batch_size'], WORD_SORT_K_BATCHES, training=False) tgt_bdict = {v: k for k, v in tgt_dict.iteritems()} # finally, run decoding decoded_sentences = list() decode_costs = [] lengths = [] logger.info('starting decoding') tic = time.time() sent_ind = 0 for ss, sm, ts, tm in eval_stream.get_epoch_iterator():