'--resume_epoch',
type=int,
default=None,
help=
'resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)'
)
args = parser.parse_args()
args.max_seq_len = WORD_MAX_SEQ_LEN
args.sort_k_batches = WORD_SORT_K_BATCHES
logger, opts = setup_exp(args)
logger.info('loading data...')
train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE,
TGT_TRAIN_FILE,
args.batch_size,
WORD_SORT_K_BATCHES,
training=True)
logger.info('done loading data')
src_bdict = {v: k for k, v in src_dict.iteritems()}
tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
valid_stream, _, _ = load_data(SRC_VALID_FILE,
TGT_VALID_FILE,
args.batch_size,
WORD_SORT_K_BATCHES,
training=False)
if args.resume_epoch:
assert (args.resume_epoch > 0)
start_epoch = args.resume_epoch
logger.info('loading model...')
help='insertion bonus')
args = parser.parse_args()
args.reverse = True # just in case
logger = get_logger(args.expdir)
def model_fn(x):
return EncoderDecoder(x)
model, opts = restore_model_and_opts(model_fn, args, logger)
# TODO try w/ test as well
if not args.test:
eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE,
TGT_VALID_FILE,
opts['batch_size'],
WORD_SORT_K_BATCHES,
training=False)
else:
eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE,
TGT_TEST_FILE,
opts['batch_size'],
WORD_SORT_K_BATCHES,
training=False)
tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
# finally, run decoding
decoded_sentences = list()
decode_costs = []
parser.add_argument('--dropout', type=float, default=0.0, help='dropout (fraction of units randomly dropped on non-recurrent connections)')
parser.add_argument('--src_vocab_size', type=int, default=SRC_VOCAB_SIZE, help='source vocabulary size')
parser.add_argument('--tgt_vocab_size', type=int, default=TGT_VOCAB_SIZE, help='source vocabulary size')
parser.add_argument('--epochs', type=int, default=8, help='number of epochs to train')
parser.add_argument('--print_every', type=int, default=1, help='how often to print cost')
parser.add_argument('--optimizer', type=str, default='sgd', choices=optimizers)
parser.add_argument('--reverse', action='store_true', help='reverse source input sentence')
parser.add_argument('--expdir', type=str, default='sandbox', help='experiment directory to save files to')
parser.add_argument('--resume_epoch', type=int, default=None, help='resume starting at this epoch (expdir must have saved models, and this will also load previous opts.json for configuration)')
args = parser.parse_args()
args.max_seq_len = WORD_MAX_SEQ_LEN; args.sort_k_batches = WORD_SORT_K_BATCHES
logger, opts = setup_exp(args)
logger.info('loading data...')
train_stream, src_dict, tgt_dict = load_data(SRC_TRAIN_FILE, TGT_TRAIN_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=True)
logger.info('done loading data')
src_bdict = {v: k for k, v in src_dict.iteritems()}
tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
valid_stream, _, _ = load_data(SRC_VALID_FILE, TGT_VALID_FILE, args.batch_size, WORD_SORT_K_BATCHES, training=False)
if args.resume_epoch:
assert (args.resume_epoch > 0)
start_epoch = args.resume_epoch
logger.info('loading model...')
encdec = EncoderDecoder(args)
load_model_params(encdec, pjoin(args.expdir, 'model_epoch%d.h5' % (args.resume_epoch - 1)))
logger.info('done loading model')
else:
start_epoch = 0
encdec = EncoderDecoder(args)
parser.add_argument('expdir', type=str, help='experiment directory')
parser.add_argument('model', type=str, help='model file to use')
parser.add_argument('outfile', type=str, help='file to write decoder output setnences to')
parser.add_argument('--beam', type=int, default=12, help='beam size to use')
parser.add_argument('--test', action='store_true', help='evaluate on test set instead of validation set')
parser.add_argument('--beta', type=float, default=1.0, help='insertion bonus')
args = parser.parse_args()
args.reverse = True # just in case
logger = get_logger(args.expdir)
def model_fn(x): return EncoderDecoder(x)
model, opts = restore_model_and_opts(model_fn, args, logger)
# TODO try w/ test as well
if not args.test:
eval_stream, src_dict, tgt_dict = load_data(SRC_VALID_FILE, TGT_VALID_FILE,
opts['batch_size'], WORD_SORT_K_BATCHES, training=False)
else:
eval_stream, src_dict, tgt_dict = load_data(SRC_TEST_FILE, TGT_TEST_FILE,
opts['batch_size'], WORD_SORT_K_BATCHES, training=False)
tgt_bdict = {v: k for k, v in tgt_dict.iteritems()}
# finally, run decoding
decoded_sentences = list()
decode_costs = []
lengths = []
logger.info('starting decoding')
tic = time.time()
sent_ind = 0
for ss, sm, ts, tm in eval_stream.get_epoch_iterator():
