""" Sanity Check """ if args.model == "dictattn": if not args.dict: raise ValueError("When using dict attn, you need to specify the (--dict) lexical dictionary files.") else: if args.dict: raise ValueError("When not using dict attn, you do not need to specify the dictionary.") if args.use_cpu: args.gpu = -1 if args.save_models: args.save_len = 1 """ Training """ trainer = ParallelTrainer(args.seed, args.gpu) # data UF.trace("Loading corpus + dictionary") with open(args.src) as src_fp: with open(args.trg) as trg_fp: SRC, TRG, train_data = load_nmt_train_data(src_fp, trg_fp, cut_threshold=args.unk_cut) train_data = list(batch_generator(train_data, (SRC, TRG), args.batch)) UF.trace("SRC size:", len(SRC)) UF.trace("TRG size:", len(TRG)) UF.trace("Data loaded.") # dev data dev_data = None if args.src_dev and args.trg_dev: with open(args.src_dev) as src_fp:
parser.add_argument("--verbose", action="store_true", help="To output the training progress for every sentence in corpora.") parser.add_argument("--use_cpu", action="store_true", help="Force to use CPU.") parser.add_argument("--save_models", action="store_true", help="Save models for every iteration with auto enumeration.") parser.add_argument("--gpu", type=int, default=-1, help="Specify GPU to be used, negative for using CPU.") parser.add_argument("--init_model", type=str, help="Init the training weights with saved model.") parser.add_argument("--model",type=str,choices=["lstm"], default="lstm", help="Type of model being trained.") parser.add_argument("--unk_cut", type=int, default=1, help="Threshold for words in corpora to be treated as unknown.") parser.add_argument("--dropout", type=positive_decimal, default=0.2, help="Dropout ratio for LSTM.") parser.add_argument("--seed", type=int, default=0, help="Seed for RNG. 0 for totally random seed.") args = parser.parse_args() if args.use_cpu: args.gpu = -1 """ Training """ trainer = ParallelTrainer(args.seed, args.gpu) # data UF.trace("Loading corpus + dictionary") X, Y, data = load_pos_train_data(sys.stdin, cut_threshold=args.unk_cut) data = list(batch_generator(data, (X, Y), args.batch)) UF.trace("INPUT size:", len(X)) UF.trace("LABEL size:", len(Y)) UF.trace("Data loaded.") """ Setup model """ UF.trace("Setting up classifier") opt = optimizers.Adam() model = ParallelTextClassifier(args, X, Y, opt, args.gpu, activation=F.relu, collect_output=args.verbose) """ Training Callback """