return opt
opt = main()
####################### Output path, logger, device and random seed configuration #################
exp_path = opt.read_model_path if opt.testing else hyperparam_pseudo_method(
opt)
if not os.path.exists(exp_path):
os.makedirs(exp_path)
logger = set_logger(exp_path, testing=opt.testing)
logger.info("Parameters: " + str(json.dumps(vars(opt), indent=4)))
logger.info("Experiment path: %s" % (exp_path))
sp_device, qg_device = set_torch_device(opt.deviceId[0]), set_torch_device(
opt.deviceId[1])
set_random_seed(opt.seed, device='cuda')
################################ Vocab and Data Reader ###########################
sp_copy, qg_copy = 'copy__' in opt.read_sp_model_path, 'copy__' in opt.read_qg_model_path
sp_vocab, qg_vocab = Vocab(opt.dataset, task='semantic_parsing',
copy=sp_copy), Vocab(opt.dataset,
task='question_generation',
copy=qg_copy)
logger.info("Semantic Parsing model vocabulary ...")
logger.info("Vocab size for input natural language sentence is: %s" %
(len(sp_vocab.word2id)))
logger.info("Vocab size for output logical form is: %s" %
(len(sp_vocab.lf2id)))
if opt.testing:
assert opt.read_model_path
return opt
opt = main()
####################### Output path, logger, device and random seed configuration #################
exp_path = opt.read_model_path if opt.testing else hyperparam_lm(opt)
if not os.path.exists(exp_path):
os.makedirs(exp_path)
logger = set_logger(exp_path, testing=opt.testing)
logger.info("Parameters: " + str(json.dumps(vars(opt), indent=4)))
logger.info("Experiment path: %s" % (exp_path))
opt.device = set_torch_device(opt.deviceId)
set_random_seed(opt.seed, device=opt.device.type)
################################ Vocab and Data Reader ###########################
lm_vocab = Vocab(opt.dataset, task='language_model')
if opt.side == 'question':
word2id = lm_vocab.word2id
logger.info("Vocab size for natural language sentence is: %s" % (len(word2id)))
else:
word2id = lm_vocab.lf2id
logger.info("Vocab size for logical form is: %s" % (len(word2id)))
logger.info("Read dataset %s starts at %s" % (opt.dataset, time.asctime(time.localtime(time.time()))))
Example.set_domain(opt.dataset)
if not opt.testing:
parser.add_argument('--max_epoch', default=100, type=int)
parser.add_argument('--deviceId',
type=int,
default=-1,
help='train model on ith gpu. -1:cpu, 0:auto_select')
parser.add_argument('--seed', type=int, default=999)
args = parser.parse_args()
assert (not args.testing) or args.read_model_path
if args.testing:
exp_path = args.read_model_path
else:
exp_path = set_hyperparam_path(args)
if not os.path.exists(exp_path):
os.makedirs(exp_path)
logger = set_logger(exp_path, testing=args.testing)
device, args.deviceId = set_torch_device(args.deviceId)
set_random_seed(args.seed, device=device)
logger.info("Parameters:" + str(json.dumps(vars(args), indent=4)))
logger.info("Experiment path: %s" % (exp_path))
logger.info("Read dataset starts at %s" %
(time.asctime(time.localtime(time.time()))))
start_time = time.time()
Example.set_tokenizer(args.bert) # set bert tokenizer
if not args.testing:
train_loader = DataLoader(QNLIDataset('train'),
batch_size=args.batch_size,
shuffle=True,
collate_fn=collate_fn_labeled)
dev_loader = DataLoader(QNLIDataset('dev'),
help='train model on ith gpu. -1:cpu, 0:auto_select')
parser.add_argument('--ngpus',
type=int,
default=1,
help='use multi gpus to train model')
parser.add_argument('--seed', type=int, default=999)
args = parser.parse_args()
assert (not args.testing) or args.read_model_path
if args.testing:
exp_path = args.read_model_path
else:
exp_path = set_hyperparam_path(args)
if not os.path.exists(exp_path):
os.makedirs(exp_path)
logger = set_logger(exp_path, testing=args.testing)
device, deviceIdList = set_torch_device(args.deviceId, args.ngpus)
set_random_seed(args.seed, device=device.type)
logger.info("Parameters:" + str(json.dumps(vars(args), indent=4)))
logger.info("Experiment path: %s" % (exp_path))
logger.info("Read dataset starts at %s" %
(time.asctime(time.localtime(time.time()))))
start_time = time.time()
Example.set_tokenizer(args.bert) # set bert tokenizer
if not args.testing:
train_loader = DataLoader(QNLIDataset('train'),
batch_size=args.batch_size * args.ngpus,
shuffle=True,
collate_fn=collate_fn_labeled,
num_workers=4)