def generator_sample_example(self, positive_dir, negative_dir, num_batch):
self.temp_positive_dir = positive_dir
self.temp_negative_dir = negative_dir
if not os.path.exists(self.temp_positive_dir): os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir): os.mkdir(self.temp_negative_dir)
shutil.rmtree(self.temp_negative_dir)
shutil.rmtree(self.temp_positive_dir)
if not os.path.exists(self.temp_positive_dir): os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir): os.mkdir(self.temp_negative_dir)
counter = 0
for i in range(num_batch):
decode_result = self._model.run_eval_given_step(self._sess, self.batches[self.current_batch])
for i in range(FLAGS.batch_size):
decoded_words_all = []
original_review = self.batches[self.current_batch].original_review_output[i]
for j in range(FLAGS.max_dec_sen_num):
output_ids = [int(t) for t in decode_result['generated'][i][j]][1:]
decoded_words = data.outputids2words(output_ids, self._vocab, None)
# Remove the [STOP] token from decoded_words, if necessary
try:
fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
if len(decoded_words)<2:
continue
if len(decoded_words_all)>0:
new_set1 =set(decoded_words_all[len(decoded_words_all)-1].split())
new_set2= set(decoded_words)
if len(new_set1 & new_set2) > 0.5 * len(new_set2):
continue
decoded_output = ' '.join(decoded_words).strip() # single string
decoded_words_all.append(decoded_output)
decoded_words_all = ' '.join(decoded_words_all).strip()
try:
fst_stop_idx = decoded_words_all.index(
data.STOP_DECODING_DOCUMENT) # index of the (first) [STOP] symbol
decoded_words_all = decoded_words_all[:fst_stop_idx]
except ValueError:
decoded_words_all = decoded_words_all
decoded_words_all = decoded_words_all.replace("[UNK] ", "")
decoded_words_all = decoded_words_all.replace("[UNK]", "")
decoded_words_all, _ = re.subn(r"(! ){2,}", "! ", decoded_words_all)
decoded_words_all, _ = re.subn(r"(\. ){2,}", ". ", decoded_words_all)
self.write_negtive_temp_to_json(original_review, decoded_words_all, counter)
counter += 1 # this is how many examples we've decoded
self.current_batch +=1
if self.current_batch >= len(self.batches):
self.current_batch = 0
eva = Evaluate()
eva.diversity_evaluate(negative_dir + "/*")
def generator_sample_example(self, positive_dir, negative_dir, num_batch):
self.temp_positive_dir = positive_dir
self.temp_negative_dir = negative_dir
if not os.path.exists(self.temp_positive_dir): os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir): os.mkdir(self.temp_negative_dir)
shutil.rmtree(self.temp_negative_dir)
shutil.rmtree(self.temp_positive_dir)
if not os.path.exists(self.temp_positive_dir): os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir): os.mkdir(self.temp_negative_dir)
counter = 0
for i in range(num_batch):
decode_result = self._model.run_eval_given_step(self._sess, self.batches[self.current_batch])
for i in range(FLAGS.batch_size):
decoded_words_all = []
original_review = self.batches[self.current_batch].original_review_output[i]
for j in range(FLAGS.max_dec_sen_num):
output_ids = [int(t) for t in decode_result['generated'][i][j]][1:]
decoded_words = data.outputids2words(output_ids, self._vocab, None)
# Remove the [STOP] token from decoded_words, if necessary
try:
fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
if len(decoded_words)<2:
continue
if len(decoded_words_all)>0:
new_set1 =set(decoded_words_all[len(decoded_words_all)-1].split())
new_set2= set(decoded_words)
if len(new_set1 & new_set2) > 0.5 * len(new_set2):
continue
decoded_output = ' '.join(decoded_words).strip() # single string
decoded_words_all.append(decoded_output)
decoded_words_all = ' '.join(decoded_words_all).strip()
try:
fst_stop_idx = decoded_words_all.index(
data.STOP_DECODING_DOCUMENT) # index of the (first) [STOP] symbol
decoded_words_all = decoded_words_all[:fst_stop_idx]
except ValueError:
decoded_words_all = decoded_words_all
decoded_words_all = decoded_words_all.replace("[UNK] ", "")
decoded_words_all = decoded_words_all.replace("[UNK]", "")
decoded_words_all, _ = re.subn(r"(! ){2,}", "! ", decoded_words_all)
decoded_words_all, _ = re.subn(r"(\. ){2,}", ". ", decoded_words_all)
self.write_negtive_temp_to_json(original_review, decoded_words_all, counter)
counter += 1 # this is how many examples we've decoded
self.current_batch +=1
if self.current_batch >= len(self.batches):
self.current_batch = 0
eva = Evaluate()
eva.diversity_evaluate(negative_dir + "/*")
def generator_test_max_example(self, positive_dir, negative_dir,
num_batch):
self.temp_positive_dir = positive_dir
self.temp_negative_dir = negative_dir
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir):
os.mkdir(self.temp_negative_dir)
shutil.rmtree(self.temp_negative_dir)
shutil.rmtree(self.temp_positive_dir)
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negative_dir):
os.mkdir(self.temp_negative_dir)
counter = 0
batches = self.test_batches
step = 0
list_hop = []
list_ref = []
while step < num_batch:
batch = batches[step]
step += 1
decode_result = self._model.max_generator(self._sess, batch)
for i in range(FLAGS.batch_size):
decoded_words_all = []
original_review = batch.original_review_output[i]
for j in range(FLAGS.max_dec_sen_num):
output_ids = [
int(t) for t in decode_result['generated'][i][j]
][1:]
decoded_words = data.outputids2words(
output_ids, self._vocab, None)
# Remove the [STOP] token from decoded_words, if necessary
try:
fst_stop_idx = decoded_words.index(
data.STOP_DECODING
) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
if len(decoded_words) < 2:
continue
if len(decoded_words_all) > 0:
new_set1 = set(
decoded_words_all[len(decoded_words_all) -
1].split())
new_set2 = set(decoded_words)
if len(new_set1 & new_set2) > 0.5 * len(new_set2):
continue
decoded_output = ' '.join(
decoded_words).strip() # single string
decoded_words_all.append(decoded_output)
decoded_words_all = ' '.join(decoded_words_all).strip()
try:
fst_stop_idx = decoded_words_all.index(
data.STOP_DECODING_DOCUMENT
) # index of the (first) [STOP] symbol
decoded_words_all = decoded_words_all[:fst_stop_idx]
except ValueError:
decoded_words_all = decoded_words_all
decoded_words_all = decoded_words_all.replace("[UNK] ", "")
decoded_words_all = decoded_words_all.replace("[UNK]", "")
decoded_words_all, _ = re.subn(r"(! ){2,}", "! ",
decoded_words_all)
decoded_words_all, _ = re.subn(r"(\. ){2,}", ". ",
decoded_words_all)
self.write_negtive_temp_to_json(
batch.original_review_inputs[i], original_review,
decoded_words_all, counter)
list_ref.append([nltk.word_tokenize(original_review)])
list_hop.append(nltk.word_tokenize(decoded_words_all))
counter += 1 # this is how many examples we've decoded
# bleu_score = corpus_bleu(list_ref, list_hop)
# tf.logging.info('bleu: ' + str(bleu_score))
eva = Evaluate()
eva.diversity_evaluate(negative_dir + "/*")
def generator_max_example(self, target_batches, positive_dir,
negetive_dir):
self.temp_positive_dir = positive_dir
self.temp_negetive_dir = negetive_dir
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negetive_dir):
os.mkdir(self.temp_negetive_dir)
shutil.rmtree(self.temp_negetive_dir)
shutil.rmtree(self.temp_positive_dir)
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negetive_dir):
os.mkdir(self.temp_negetive_dir)
counter = 0
batches = target_batches
step = 0
while step < len(target_batches):
batch = copy.deepcopy(batches[step])
step += 1
decoded_words_all = [[] for i in range(FLAGS.batch_size)]
original_reviews = [
" ".join(review) for review in batch.orginal_all_text
]
#tf.logging.info(batch.enc_lens)
for k in range(FLAGS.max_dec_sen_num):
decode_result = self._model.max_generator(self._sess, batch)
srl_batch, decode_mask = self.output_to_batch(
batch, decode_result)
decode_result_seq = self._srl_model.max_generator(
self._sess_srl, srl_batch)
if k < FLAGS.max_dec_sen_num - 1:
batch = self.seq_output_to_batch(decode_result_seq, batch)
for i in range(FLAGS.batch_size):
if decode_mask[i] == 0:
decoded_words_all[i].append(
data.STOP_DECODING_DOCUMENT)
else:
output_ids = [
int(t) for t in decode_result_seq['generated'][i]
][0:]
decoded_words = data.outputids2words(
output_ids, self._vocab, None)
# Remove the [STOP] token from decoded_words, if necessary
try:
fst_stop_idx = decoded_words.index(
data.STOP_DECODING
) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
if len(decoded_words) < 1:
continue
if len(decoded_words_all[i]) > 0:
new_set1 = set(
decoded_words_all[i][len(decoded_words_all[i])
- 1].split())
new_set2 = set(decoded_words)
if len(new_set1 & new_set2) > 0.5 * len(new_set2):
continue
decoded_output = ' '.join(
decoded_words).strip() # single string
decoded_words_all[i].append(decoded_output)
for i in range(FLAGS.batch_size):
batch_seq = ' '.join([
decoded_words_all[i][j]
for j in range(len(decoded_words_all[i]))
]).strip()
try:
fst_stop_idx = batch_seq.index(
data.STOP_DECODING_DOCUMENT
) # index of the (first) [STOP] symbol
batch_seq = batch_seq[:fst_stop_idx]
except ValueError:
batch_seq = batch_seq
batch_seq = batch_seq.replace("[UNK] ", "")
batch_seq = batch_seq.replace("[UNK]", "")
batch_seq, _ = re.subn(r"(! ){2,}", "! ", batch_seq)
batch_seq, _ = re.subn(r"(\. ){2,}", ". ", batch_seq)
self.write_negtive_temp_to_json(positive_dir, negetive_dir,
original_reviews[i], batch_seq)
eva = Evaluate()
eva.diversity_evaluate(negetive_dir + "/*")
def main(unused_argv):
if len(unused_argv) != 1: # prints a message if you've entered flags incorrectly
raise Exception("Problem with flags: %s" % unused_argv)
tf.logging.set_verbosity(tf.logging.INFO) # choose what level of logging you want
tf.logging.info('Starting running in %s mode...', (FLAGS.mode))
# Change log_root to FLAGS.log_root/FLAGS.exp_name and create the dir if necessary
FLAGS.log_root = os.path.join(FLAGS.log_root, FLAGS.exp_name)
if not os.path.exists(FLAGS.log_root):
if FLAGS.mode=="train":
os.makedirs(FLAGS.log_root)
else:
raise Exception("Logdir %s doesn't exist. Run in train mode to create it." % (FLAGS.log_root))
vocab = Vocab(FLAGS.vocab_path, FLAGS.vocab_size) # create a vocabulary
# Make a namedtuple hps, containing the values of the hyperparameters that the model needs
hparam_list = ['mode', 'lr', 'adagrad_init_acc', 'rand_unif_init_mag', 'trunc_norm_init_std', 'max_grad_norm', 'hidden_dim', 'emb_dim', 'batch_size', 'max_dec_sen_num','max_dec_steps', 'max_enc_steps']
hps_dict = {}
for key,val in FLAGS.__flags.items(): # for each flag
if key in hparam_list: # if it's in the list
hps_dict[key] = val # add it to the dict
hps_generator = namedtuple("HParams", hps_dict.keys())(**hps_dict)
hparam_list = ['lr', 'adagrad_init_acc', 'rand_unif_init_mag', 'trunc_norm_init_std', 'max_grad_norm',
'hidden_dim', 'emb_dim', 'batch_size', 'max_enc_sen_num', 'max_enc_seq_len']
hps_dict = {}
for key, val in FLAGS.__flags.items(): # for each flag
if key in hparam_list: # if it's in the list
hps_dict[key] = val # add it to the dict
hps_discriminator = namedtuple("HParams", hps_dict.keys())(**hps_dict)
# Create a batcher object that will create minibatches of data
batcher = GenBatcher(vocab, hps_generator)
tf.set_random_seed(111) # a seed value for randomness
if hps_generator.mode == 'train':
print("Start pre-training......")
model = Generator(hps_generator, vocab)
sess_ge, saver_ge, train_dir_ge = setup_training_generator(model)
generated = Generated_sample(model, vocab, batcher, sess_ge)
print("Start pre-training generator......")
run_pre_train_generator(model, batcher, 10, sess_ge, saver_ge, train_dir_ge,generated) # this is an infinite loop until
print("Generating negetive examples......")
generated.generator_whole_negetive_example()
generated.generator_test_negetive_example()
model_dis = Discriminator(hps_discriminator, vocab)
dis_batcher = DisBatcher(hps_discriminator, vocab, "train/generated_samples_positive/*", "train/generated_samples_negetive/*", "test/generated_samples_positive/*", "test/generated_samples_negetive/*")
sess_dis, saver_dis, train_dir_dis = setup_training_discriminator(model_dis)
print("Start pre-training discriminator......")
#run_test_discriminator(model_dis, dis_batcher, sess_dis, saver_dis, "test")
run_pre_train_discriminator(model_dis, dis_batcher, 25, sess_dis, saver_dis, train_dir_dis)
util.load_ckpt(saver_ge, sess_ge, ckpt_dir="train-generator")
generated.generator_sample_example("sample_temp_positive", "sample_temp_negetive", 1000)
generated.generator_test_sample_example("test_sample_temp_positive",
"test_sample_temp_negetive",
200)
generated.generator_test_max_example("test_max_temp_positive",
"test_max_temp_negetive",
200)
tf.logging.info("true data diversity: ")
eva = Evaluate()
eva.diversity_evaluate("test_sample_temp_positive" + "/*")
print("Start adversial training......")
whole_decay = False
for epoch in range(1):
batches = batcher.get_batches(mode='train')
for step in range(int(len(batches)/1000)):
run_train_generator(model,model_dis,sess_dis,batcher,dis_batcher,batches[step*1000:(step+1)*1000],sess_ge, saver_ge, train_dir_ge,generated) #(model, discirminator_model, discriminator_sess, batcher, dis_batcher, batches, sess, saver, train_dir, generated):
generated.generator_sample_example("sample_generated/"+str(epoch)+"epoch_step"+str(step)+"_temp_positive", "sample_generated/"+str(epoch)+"epoch_step"+str(step)+"_temp_negetive", 1000)
#generated.generator_max_example("max_generated/"+str(epoch)+"epoch_step"+str(step)+"_temp_positive", "max_generated/"+str(epoch)+"epoch_step"+str(step)+"_temp_negetive", 200)
tf.logging.info("test performance: ")
tf.logging.info("epoch: "+str(epoch)+" step: "+str(step))
generated.generator_test_sample_example(
"test_sample_generated/" + str(epoch) + "epoch_step" + str(step) + "_temp_positive",
"test_sample_generated/" + str(epoch) + "epoch_step" + str(step) + "_temp_negetive", 200)
generated.generator_test_max_example("test_max_generated/" + str(epoch) + "epoch_step" + str(step) + "_temp_positive",
"test_max_generated/" + str(epoch) + "epoch_step" + str(step) + "_temp_negetive",
200)
dis_batcher.train_queue = []
dis_batcher.train_queue = []
for i in range(epoch+1):
for j in range(step+1):
dis_batcher.train_queue += dis_batcher.fill_example_queue("sample_generated/"+str(i)+"epoch_step"+str(j)+"_temp_positive/*")
dis_batcher.train_queue += dis_batcher.fill_example_queue("sample_generated/"+str(i)+"epoch_step"+str(j)+"_temp_negetive/*")
dis_batcher.train_batch = dis_batcher.create_batches(mode="train", shuffleis=True)
#dis_batcher.valid_batch = dis_batcher.train_batch
whole_decay = run_train_discriminator(model_dis, 5, dis_batcher, dis_batcher.get_batches(mode="train"),
sess_dis, saver_dis, train_dir_dis, whole_decay)
'''elif hps_generator.mode == 'decode':
decode_model_hps = hps_generator # This will be the hyperparameters for the decoder model
model = Generator(decode_model_hps, vocab)
generated = Generated_sample(model, vocab, batcher)
bleu_score = generated.compute_BLEU()'=
tf.logging.info('bleu: %f', bleu_score) # print the loss to screen'''
else:
def generator_max_example(self, target_batches, positive_dir,
negetive_dir):
self.temp_positive_dir = positive_dir
self.temp_negetive_dir = negetive_dir
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negetive_dir):
os.mkdir(self.temp_negetive_dir)
shutil.rmtree(self.temp_negetive_dir)
shutil.rmtree(self.temp_positive_dir)
if not os.path.exists(self.temp_positive_dir):
os.mkdir(self.temp_positive_dir)
if not os.path.exists(self.temp_negetive_dir):
os.mkdir(self.temp_negetive_dir)
counter = 0
batches = target_batches
step = 0
while step < len(target_batches):
batch = batches[step]
step += 1
decode_result = self._model.max_generator(self._sess, batch)
for i in range(FLAGS.batch_size):
original_review = batch.orig_outputs[i]
output_ids = [int(t)
for t in decode_result['generated'][i]][0:]
decoded_words = data.outputids2words(output_ids, self._vocab,
None)
# Remove the [STOP] token from decoded_words, if necessary
try:
fst_stop_idx = decoded_words.index(
data.STOP_DECODING
) # index of the (first) [STOP] symbol
decoded_words = decoded_words[:fst_stop_idx]
except ValueError:
decoded_words = decoded_words
decoded_output = ' '.join(
decoded_words).strip() # single string
try:
fst_stop_idx = decoded_output.index(
data.STOP_DECODING_DOCUMENT
) # index of the (first) [STOP] symbol
decoded_output = decoded_output[:fst_stop_idx]
except ValueError:
decoded_output = decoded_output
decoded_output = decoded_output.replace("[UNK] ", "")
decoded_output = decoded_output.replace("[UNK]", "")
decoded_output, _ = re.subn(r"(! ){2,}", "! ",
decoded_output)
decoded_output, _ = re.subn(r"(\. ){2,}", ". ",
decoded_output)
self.write_negtive_temp_to_json(positive_dir, negetive_dir,
original_review,
decoded_output)
eva = Evaluate()
eva.diversity_evaluate(negetive_dir + "/*")