def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = LanguageGeneration
data_arg = Storage()
data_arg.file_id = args.dataid
data_arg.tokenizer = args.tokenizer
data_arg.max_sent_length = args.max_sent_length
data_arg.convert_to_lower_letter = args.convert_to_lower_letter
data_arg.min_frequent_vocab_times = args.min_frequent_vocab_times
data_arg.min_rare_vocab_times = args.min_rare_vocab_times
wordvec_class = GeneralWordVector
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load_matrix(embedding_size, dm.frequent_vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = TransformerLM(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
json.dump(test_res, open("./result.json", "w"))
elif args.mode == "load":
return model
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load(embedding_size, dm.vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
for key, val in test_res.items():
if isinstance(val, bytes):
test_res[key] = str(val)
json.dump(test_res, open("./result.json", "w"))
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = LanguageGeneration
data_arg = Storage()
data_arg.file_id = args.dataid
data_arg.max_sent_length = args.max_sent_length
data_arg.convert_to_lower_letter = args.convert_to_lower_letter
data_arg.pretrained = args.pretrained
data_arg.tokenizer = args.pretrained_model
def load_dataset(data_arg):
tokenizer = PretrainedTokenizer(
GPT2Tokenizer.from_pretrained(data_arg.tokenizer))
new_arg = Storage(data_arg.copy())
new_arg.tokenizer = tokenizer
dm = data_class(**new_arg)
return dm
if args.cache:
dm = try_cache(load_dataset, (data_arg, ), args.cache_dir,
data_class.__name__)
else:
dm = load_dataset(data_arg)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = GPT2LM(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
json.dump(test_res, open("./result.json", "w"))
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath + "#OpenSubtitles"
data_arg.tokenizer = PretrainedTokenizer(
BertTokenizer.from_pretrained(args.bert_vocab))
data_arg.pretrained = "bert"
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load_matrix(embedding_size, dm.frequent_vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
else:
raise ValueError("Unknown mode")
def main(args):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
data_class = SkeletonGeneration
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
if args.cache:
dm = try_cache(data_class, (args.datapath, ), args.cache_dir)
volatile.wordvec = try_cache(\
lambda wv, ez, vl: wordvec_class(wv).load(ez, vl), \
(args.wvpath, args.embedding_size, dm.vocab_list),
args.cache_dir, wordvec_class.__name__)
else:
dm = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
volatile.wordvec = wv.load(args.embedding_size, dm.vocab_list)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = LM(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
else:
raise ValueError("Unknown mode")
def main(args):
logging.basicConfig(
filename=0,
level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(args.cuda_num, args.cuda)
volatile = Storage()
volatile.load_exclude_set = args.load_exclude_set
volatile.restoreCallback = args.restoreCallback
if args.dataset == 'WizardOfWiki':
data_class = WizardOfWiki
elif args.dataset == 'HollE':
data_class = HollE
else:
raise ValueError
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
if not os.path.exists(args.cache_dir):
os.mkdir(args.cache_dir)
args.cache_dir = os.path.join(args.cache_dir, args.dataset)
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
args.out_dir = os.path.join(args.out_dir, args.dataset)
if not os.path.exists(args.model_dir):
os.mkdir(args.model_dir)
if args.dataset not in args.model_dir:
args.model_dir = os.path.join(args.model_dir, args.dataset)
if args.cache:
dm = try_cache(data_class, (args.datapath, ), args.cache_dir)
volatile.wordvec = try_cache(
lambda wv, ez, vl: wordvec_class(wv).load_matrix(ez, vl),
(args.wvpath, args.embedding_size, dm.vocab_list), args.cache_dir,
wordvec_class.__name__)
else:
dm = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
volatile.wordvec = wv.load_matrix(args.embedding_size, dm.vocab_list)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
elif args.mode == 'dev':
model.test_dev()
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(
filename=0,
level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(args.device, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath
# RAML parameters
if args.model == "raml":
data_arg.raml_file = "samples_iwslt14.txt"
data_arg.num_samples = 10 or args.n_samples
data_arg.tau = 0.4
wordvec_class = WordVector.load_class(args.wvclass)
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load_matrix(embedding_size, dm.vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
if args.model == "basic":
model = Seq2seq(param)
elif args.model == "raml":
model = Seq2seqRAML(param)
elif args.model == "scheduled-sampling":
model = Seq2seqSS(param)
elif args.model == "policy-gradient":
model = Seq2seqPG(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
json.dump(test_res, open("./result.json", "w"))
else:
raise ValueError("Unknown mode")