help='report interval')
parser.add_argument('--save_dir', type=str, default='transformer_out',
help='directory path to save the final model and training log')
parser.add_argument('--gpus', type=str,
help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu.'
'(using single gpu is suggested)')
parser.add_argument('--model_parameter', type=str, default=' ', required=True,
help='model parameter for inference, must be provided.')
args = parser.parse_args()
logging_config(args.save_dir)
logging.info(args)
# data process
data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab \
= dataprocessor.load_translation_data(dataset=args.dataset, bleu=args.bleu, args=args)
dataprocessor.write_sentences(test_tgt_sentences, os.path.join(args.save_dir, 'test_gt.txt'))
data_train = data_train.transform(lambda src, tgt: (src, tgt, len(src), len(tgt)), lazy=False)
data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_val)])
data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_test)])
data_train_lengths, data_val_lengths, data_test_lengths = [dataprocessor.get_data_lengths(x)
for x in
[data_train, data_val, data_test]]
detokenizer = nlp.data.SacreMosesDetokenizer()
' as official script (mteval-v13a.pl) used by WMT; '
'"intl": This use international tokenization in mteval-v14a.pl')
parser.add_argument('--log_interval', type=int, default=100, metavar='N',
help='report interval')
parser.add_argument('--save_dir', type=str, default='transformer_out',
help='directory path to save the final model and training log')
parser.add_argument('--gpus', type=str,
help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu.'
'(using single gpu is suggested)')
args = parser.parse_args()
logging_config(args.save_dir)
logging.info(args)
data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab \
= dataprocessor.load_translation_data(dataset=args.dataset, bleu=args.bleu, args=args)
dataprocessor.write_sentences(val_tgt_sentences, os.path.join(args.save_dir, 'val_gt.txt'))
dataprocessor.write_sentences(test_tgt_sentences, os.path.join(args.save_dir, 'test_gt.txt'))
data_train = data_train.transform(lambda src, tgt: (src, tgt, len(src), len(tgt)), lazy=False)
data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_val)])
data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_test)])
ctx = [mx.cpu()] if args.gpus is None or args.gpus == '' else \
[mx.gpu(int(x)) for x in args.gpus.split(',')]
data_train_lengths, data_val_lengths, data_test_lengths = [dataprocessor.get_data_lengths(x)
for x in
parser.add_argument(
'--gpu',
type=int,
default=None,
help='id of the gpu to use. Set it to empty means to use cpu.')
parser.add_argument('--validate_on_test_data',
type=bool,
default=False,
help='To perform validation on test data')
args = parser.parse_args()
print(args)
logging_config(args.save_dir)
data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\
= dataprocessor.load_translation_data(dataset=args.dataset, bleu='tweaked', args=args)
dataprocessor.write_sentences(val_tgt_sentences,
os.path.join(args.save_dir, 'val_gt.txt'))
dataprocessor.write_sentences(test_tgt_sentences,
os.path.join(args.save_dir, 'test_gt.txt'))
data_train = data_train.transform(lambda src, tgt:
(src, tgt, len(src), len(tgt)),
lazy=False)
data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]),
i) for i, ele in enumerate(data_val)])
data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]),
len(ele[1]), i)
for i, ele in enumerate(data_test)])
if args.gpu is None:
parser.add_argument('--lr_update_factor', type=float, default=0.5,
help='Learning rate decay factor')
parser.add_argument('--clip', type=float, default=5.0, help='gradient clipping')
parser.add_argument('--log_interval', type=int, default=100, metavar='N',
help='report interval')
parser.add_argument('--save_dir', type=str, default='out_dir',
help='directory path to save the final model and training log')
parser.add_argument('--gpu', type=int, default=None,
help='id of the gpu to use. Set it to empty means to use cpu.')
args = parser.parse_args()
print(args)
logging_config(args.save_dir)
data_train, data_val, data_test, val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab\
= dataprocessor.load_translation_data(dataset=args.dataset, bleu='tweaked', args=args)
dataprocessor.write_sentences(val_tgt_sentences, os.path.join(args.save_dir, 'val_gt.txt'))
dataprocessor.write_sentences(test_tgt_sentences, os.path.join(args.save_dir, 'test_gt.txt'))
data_train = data_train.transform(lambda src, tgt: (src, tgt, len(src), len(tgt)), lazy=False)
data_val = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_val)])
data_test = gluon.data.SimpleDataset([(ele[0], ele[1], len(ele[0]), len(ele[1]), i)
for i, ele in enumerate(data_test)])
if args.gpu is None:
ctx = mx.cpu()
print('Use CPU')
else:
ctx = mx.gpu(args.gpu)