Пример #1
0
""" Sanity Check """
if args.model == "dictattn":
    if not args.dict:
        raise ValueError("When using dict attn, you need to specify the (--dict) lexical dictionary files.")
else:
    if args.dict:
        raise ValueError("When not using dict attn, you do not need to specify the dictionary.")

if args.use_cpu:
    args.gpu = -1

if args.save_models:
    args.save_len = 1

""" Training """
trainer   = ParallelTrainer(args.seed, args.gpu)
 
# data
UF.trace("Loading corpus + dictionary")
with open(args.src) as src_fp:
    with open(args.trg) as trg_fp:
        SRC, TRG, train_data = load_nmt_train_data(src_fp, trg_fp, cut_threshold=args.unk_cut)
        train_data = list(batch_generator(train_data, (SRC, TRG), args.batch))
UF.trace("SRC size:", len(SRC))
UF.trace("TRG size:", len(TRG))
UF.trace("Data loaded.")

# dev data
dev_data = None
if args.src_dev and args.trg_dev:
    with open(args.src_dev) as src_fp:
Пример #2
0
parser.add_argument("--verbose", action="store_true", help="To output the training progress for every sentence in corpora.")
parser.add_argument("--use_cpu", action="store_true", help="Force to use CPU.")
parser.add_argument("--save_models", action="store_true", help="Save models for every iteration with auto enumeration.")
parser.add_argument("--gpu", type=int, default=-1, help="Specify GPU to be used, negative for using CPU.")
parser.add_argument("--init_model", type=str, help="Init the training weights with saved model.")
parser.add_argument("--model",type=str,choices=["lstm"], default="lstm", help="Type of model being trained.")
parser.add_argument("--unk_cut", type=int, default=1, help="Threshold for words in corpora to be treated as unknown.")
parser.add_argument("--dropout", type=positive_decimal, default=0.2, help="Dropout ratio for LSTM.")
parser.add_argument("--seed", type=int, default=0, help="Seed for RNG. 0 for totally random seed.")
args = parser.parse_args()

if args.use_cpu:
    args.gpu = -1

""" Training """
trainer   = ParallelTrainer(args.seed, args.gpu)

# data
UF.trace("Loading corpus + dictionary")
X, Y, data = load_pos_train_data(sys.stdin, cut_threshold=args.unk_cut)
data       = list(batch_generator(data, (X, Y), args.batch))
UF.trace("INPUT size:", len(X))
UF.trace("LABEL size:", len(Y))
UF.trace("Data loaded.")

""" Setup model """
UF.trace("Setting up classifier")
opt   = optimizers.Adam()
model = ParallelTextClassifier(args, X, Y, opt, args.gpu, activation=F.relu, collect_output=args.verbose)

""" Training Callback """