def base_test_init(self, dl):
assert isinstance(dl, WordVector)
assert isinstance(dl.file_id, str)
assert isinstance(dl.file_path, str)
WordVector.get_all_subclasses()
assert WordVector.load_class('Glove') == Glove
assert WordVector.load_class('not_subclass') == None
# initialize with none
wv = Glove(None)
assert wv.file_id == wv.file_path == None
def main(args):
if args.debug:
debug()
if args.cuda:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
else:
config = tf.ConfigProto(device_count={'GPU': 0})
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
data_class = SingleTurnDialog.load_class(args.dataset)
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class == None:
wordvec_class = Glove
if args.cache:
data = try_cache(data_class, (args.datapath, ), args.cache_dir)
vocab = data.vocab_list
embed = try_cache(lambda wv, ez, vl: wordvec_class(wv).load(ez, vl),
(args.wvpath, args.embedding_size, vocab),
args.cache_dir, wordvec_class.__name__)
else:
data = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
vocab = data.vocab_list
embed = wv.load(args.embedding_size, vocab)
embed = np.array(embed, dtype=np.float32)
with tf.Session(config=config) as sess:
model = create_model(sess, data, args, embed)
if args.mode == "train":
model.train_process(sess, data, args)
else:
model.test_process(sess, data, args)
def main(args):
if args.debug:
debug()
if args.cuda:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
else:
config = tf.ConfigProto(device_count={'GPU': 0})
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
data_class = LanguageGeneration.load_class(args.dataset)
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class == None:
wordvec_class = Glove
if args.cache:
data = try_cache(data_class, (args.datapath,), args.cache_dir)
vocab = data.vocab_list
embed = try_cache(lambda wv, ez, vl: wordvec_class(wv).load_matrix(ez, vl),
(args.wvpath, args.embedding_size, vocab),
args.cache_dir, wordvec_class.__name__)
else:
data = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
vocab = data.vocab_list
embed = wv.load_matrix(args.embedding_size, vocab)
embed = np.array(embed, dtype = np.float32)
with tf.Session(config=config) as sess:
generator, discriminator, rollout_gen = create_model(sess, data, args, embed)
if args.mode == "train":
if args.pre_train:
#Start pretraining
print('Start pre-training generator...')
generator.pre_train_process(sess, data)
print('Start pre-training discriminator...')
discriminator.train_process(generator, data, sess, args.dis_pre_epoch_num)
#Start testing
generator.test_process(sess, data)
#Start adversarial training
for batch in range(args.total_adv_batch):
print("Adversarial training %d"%batch)
print('Start adversarial training generator...')
generator.adv_train_process(sess, data, rollout_gen, discriminator)
testout = generator.pre_evaluate(sess, data, args.batch_size, "test")
if (batch % args.test_per_epoch == 0 or batch == args.total_adv_batch - 1) and batch != 0:
print('total_batch: ', batch, 'test_loss: ', testout[0])
generator.test_process(sess, data)
print('Start adversarial training discriminator...')
discriminator.train_process(generator, data, sess, args.dis_adv_epoch_num)
else:
print("Start testing...")
test_res = generator.test_process(sess, data)
for key, val in test_res.items():
if isinstance(val, bytes):
test_res[key] = str(val)
json.dump(test_res, open("./result.json", "w"))
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load(embedding_size, dm.vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
for key, val in test_res.items():
if isinstance(val, bytes):
test_res[key] = str(val)
json.dump(test_res, open("./result.json", "w"))
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath + "#OpenSubtitles"
data_arg.tokenizer = PretrainedTokenizer(
BertTokenizer.from_pretrained(args.bert_vocab))
data_arg.pretrained = "bert"
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load_matrix(embedding_size, dm.frequent_vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
else:
raise ValueError("Unknown mode")
def main(args):
if args.debug:
debug()
if args.cuda:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
else:
config = tf.ConfigProto(device_count={'GPU': 0})
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
data_class = MultiTurnDialog.load_class(args.dataset)
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class == None:
wordvec_class = Glove
if args.cache:
data = try_cache(data_class, (args.datapath, ), args.cache_dir)
vocab = data.frequent_vocab_list
embed = try_cache(
lambda wv, ez, vl: wordvec_class(wv).load_matrix(ez, vl),
(args.wvpath, args.word_embedding_size, vocab), args.cache_dir,
wordvec_class.__name__)
word2vec = try_cache(
lambda wv, ez, vl: wordvec_class(wv).load_dict(vl),
(args.wvpath, args.word_embedding_size, vocab), args.cache_dir,
wordvec_class.__name__)
else:
data = data_class(
args.datapath,
min_frequent_vocab_times=args.min_frequent_vocab_times,
max_sent_length=args.max_sent_length,
max_turn_length=args.max_turn_length)
wv = wordvec_class(args.wvpath)
vocab = data.frequent_vocab_list #dim:9508
embed = wv.load_matrix(args.word_embedding_size, vocab)
word2vec = wv.load_dict(vocab)
embed = np.array(embed, dtype=np.float32)
with tf.Session(config=config) as sess:
model = create_model(sess, data, args, embed)
if args.mode == "train":
model.train_process(sess, data, args)
else:
multi_ref_res = model.test_multi_ref(sess, data, word2vec, args)
test_res = model.test_process(sess, data, args)
test_res.update(multi_ref_res)
for key, val in test_res.items():
if isinstance(val, bytes):
test_res[key] = str(val)
json.dump(test_res, open("./result.json", "w"))
def main(args):
logging.basicConfig(\
filename=0,\
level=logging.DEBUG,\
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',\
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(0, args.cuda)
volatile = Storage()
data_class = SkeletonGeneration
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
if args.cache:
dm = try_cache(data_class, (args.datapath, ), args.cache_dir)
volatile.wordvec = try_cache(\
lambda wv, ez, vl: wordvec_class(wv).load(ez, vl), \
(args.wvpath, args.embedding_size, dm.vocab_list),
args.cache_dir, wordvec_class.__name__)
else:
dm = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
volatile.wordvec = wv.load(args.embedding_size, dm.vocab_list)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = LM(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
else:
raise ValueError("Unknown mode")
def main(args):
if args.debug:
debug()
if args.cuda:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
else:
config = tf.ConfigProto(device_count={'GPU': 0})
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
data_class = LanguageGeneration.load_class(args.dataset)
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class == None:
wordvec_class = Glove
if args.cache:
data = try_cache(data_class, (args.datapath, ), args.cache_dir)
vocab = data.vocab_list
embed = try_cache(lambda wv, ez, vl: wordvec_class(wv).load(ez, vl),
(args.wvpath, args.embedding_size, vocab),
args.cache_dir, wordvec_class.__name__)
else:
data = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
vocab = data.vocab_list
embed = wv.load(args.embedding_size, vocab)
embed = np.array(embed, dtype=np.float32)
with tf.Session(config=config) as sess:
model = create_model(sess, data, args, embed)
if args.mode == "train":
model.train_process(sess, data, args)
else:
test_res = model.test_process(sess, data, args)
for key, val in test_res.items():
if isinstance(val, bytes):
test_res[key] = str(val)
json.dump(test_res, open("./result.json", "w"))
def main(args):
logging.basicConfig(
filename=0,
level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(args.cuda_num, args.cuda)
volatile = Storage()
volatile.load_exclude_set = args.load_exclude_set
volatile.restoreCallback = args.restoreCallback
if args.dataset == 'WizardOfWiki':
data_class = WizardOfWiki
elif args.dataset == 'HollE':
data_class = HollE
else:
raise ValueError
wordvec_class = WordVector.load_class(args.wvclass)
if wordvec_class is None:
wordvec_class = Glove
if not os.path.exists(args.cache_dir):
os.mkdir(args.cache_dir)
args.cache_dir = os.path.join(args.cache_dir, args.dataset)
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
args.out_dir = os.path.join(args.out_dir, args.dataset)
if not os.path.exists(args.model_dir):
os.mkdir(args.model_dir)
if args.dataset not in args.model_dir:
args.model_dir = os.path.join(args.model_dir, args.dataset)
if args.cache:
dm = try_cache(data_class, (args.datapath, ), args.cache_dir)
volatile.wordvec = try_cache(
lambda wv, ez, vl: wordvec_class(wv).load_matrix(ez, vl),
(args.wvpath, args.embedding_size, dm.vocab_list), args.cache_dir,
wordvec_class.__name__)
else:
dm = data_class(args.datapath)
wv = wordvec_class(args.wvpath)
volatile.wordvec = wv.load_matrix(args.embedding_size, dm.vocab_list)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
model = Seq2seq(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
model.test_process()
elif args.mode == 'dev':
model.test_dev()
else:
raise ValueError("Unknown mode")
def main(args, load_exclude_set, restoreCallback):
logging.basicConfig(
filename=0,
level=logging.DEBUG,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(message)s',
datefmt='%H:%M:%S')
if args.debug:
debug()
logging.info(json.dumps(args, indent=2))
cuda_init(args.device, args.cuda)
volatile = Storage()
volatile.load_exclude_set = load_exclude_set
volatile.restoreCallback = restoreCallback
data_class = SingleTurnDialog.load_class(args.dataset)
data_arg = Storage()
data_arg.file_id = args.datapath
# RAML parameters
if args.model == "raml":
data_arg.raml_file = "samples_iwslt14.txt"
data_arg.num_samples = 10 or args.n_samples
data_arg.tau = 0.4
wordvec_class = WordVector.load_class(args.wvclass)
def load_dataset(data_arg, wvpath, embedding_size):
wv = wordvec_class(wvpath)
dm = data_class(**data_arg)
return dm, wv.load_matrix(embedding_size, dm.vocab_list)
if args.cache:
dm, volatile.wordvec = try_cache(
load_dataset, (data_arg, args.wvpath, args.embedding_size),
args.cache_dir, data_class.__name__ + "_" + wordvec_class.__name__)
else:
dm, volatile.wordvec = load_dataset(data_arg, args.wvpath,
args.embedding_size)
volatile.dm = dm
param = Storage()
param.args = args
param.volatile = volatile
if args.model == "basic":
model = Seq2seq(param)
elif args.model == "raml":
model = Seq2seqRAML(param)
elif args.model == "scheduled-sampling":
model = Seq2seqSS(param)
elif args.model == "policy-gradient":
model = Seq2seqPG(param)
if args.mode == "train":
model.train_process()
elif args.mode == "test":
test_res = model.test_process()
json.dump(test_res, open("./result.json", "w"))
else:
raise ValueError("Unknown mode")
