return opt opt = main() ####################### Output path, logger, device and random seed configuration ################# exp_path = opt.read_model_path if opt.testing else hyperparam_pseudo_method( opt) if not os.path.exists(exp_path): os.makedirs(exp_path) logger = set_logger(exp_path, testing=opt.testing) logger.info("Parameters: " + str(json.dumps(vars(opt), indent=4))) logger.info("Experiment path: %s" % (exp_path)) sp_device, qg_device = set_torch_device(opt.deviceId[0]), set_torch_device( opt.deviceId[1]) set_random_seed(opt.seed, device='cuda') ################################ Vocab and Data Reader ########################### sp_copy, qg_copy = 'copy__' in opt.read_sp_model_path, 'copy__' in opt.read_qg_model_path sp_vocab, qg_vocab = Vocab(opt.dataset, task='semantic_parsing', copy=sp_copy), Vocab(opt.dataset, task='question_generation', copy=qg_copy) logger.info("Semantic Parsing model vocabulary ...") logger.info("Vocab size for input natural language sentence is: %s" % (len(sp_vocab.word2id))) logger.info("Vocab size for output logical form is: %s" % (len(sp_vocab.lf2id)))
if opt.testing: assert opt.read_model_path return opt opt = main() ####################### Output path, logger, device and random seed configuration ################# exp_path = opt.read_model_path if opt.testing else hyperparam_lm(opt) if not os.path.exists(exp_path): os.makedirs(exp_path) logger = set_logger(exp_path, testing=opt.testing) logger.info("Parameters: " + str(json.dumps(vars(opt), indent=4))) logger.info("Experiment path: %s" % (exp_path)) opt.device = set_torch_device(opt.deviceId) set_random_seed(opt.seed, device=opt.device.type) ################################ Vocab and Data Reader ########################### lm_vocab = Vocab(opt.dataset, task='language_model') if opt.side == 'question': word2id = lm_vocab.word2id logger.info("Vocab size for natural language sentence is: %s" % (len(word2id))) else: word2id = lm_vocab.lf2id logger.info("Vocab size for logical form is: %s" % (len(word2id))) logger.info("Read dataset %s starts at %s" % (opt.dataset, time.asctime(time.localtime(time.time())))) Example.set_domain(opt.dataset) if not opt.testing:
parser.add_argument('--max_epoch', default=100, type=int) parser.add_argument('--deviceId', type=int, default=-1, help='train model on ith gpu. -1:cpu, 0:auto_select') parser.add_argument('--seed', type=int, default=999) args = parser.parse_args() assert (not args.testing) or args.read_model_path if args.testing: exp_path = args.read_model_path else: exp_path = set_hyperparam_path(args) if not os.path.exists(exp_path): os.makedirs(exp_path) logger = set_logger(exp_path, testing=args.testing) device, args.deviceId = set_torch_device(args.deviceId) set_random_seed(args.seed, device=device) logger.info("Parameters:" + str(json.dumps(vars(args), indent=4))) logger.info("Experiment path: %s" % (exp_path)) logger.info("Read dataset starts at %s" % (time.asctime(time.localtime(time.time())))) start_time = time.time() Example.set_tokenizer(args.bert) # set bert tokenizer if not args.testing: train_loader = DataLoader(QNLIDataset('train'), batch_size=args.batch_size, shuffle=True, collate_fn=collate_fn_labeled) dev_loader = DataLoader(QNLIDataset('dev'),
help='train model on ith gpu. -1:cpu, 0:auto_select') parser.add_argument('--ngpus', type=int, default=1, help='use multi gpus to train model') parser.add_argument('--seed', type=int, default=999) args = parser.parse_args() assert (not args.testing) or args.read_model_path if args.testing: exp_path = args.read_model_path else: exp_path = set_hyperparam_path(args) if not os.path.exists(exp_path): os.makedirs(exp_path) logger = set_logger(exp_path, testing=args.testing) device, deviceIdList = set_torch_device(args.deviceId, args.ngpus) set_random_seed(args.seed, device=device.type) logger.info("Parameters:" + str(json.dumps(vars(args), indent=4))) logger.info("Experiment path: %s" % (exp_path)) logger.info("Read dataset starts at %s" % (time.asctime(time.localtime(time.time())))) start_time = time.time() Example.set_tokenizer(args.bert) # set bert tokenizer if not args.testing: train_loader = DataLoader(QNLIDataset('train'), batch_size=args.batch_size * args.ngpus, shuffle=True, collate_fn=collate_fn_labeled, num_workers=4)