def _build(self):
g = tf.Graph()
with g.as_default():
# build language model
language_model = LanguageModel()
language_model.build()
language_model.set_cache_dir('test_empty')
sess = tf.Session()
language_model.set_session(sess)
language_model.setup_model()
self.language_model = language_model
def ivqa_decoding_beam_search(checkpoint_path=None):
model_config = ModelConfig()
method = FLAGS.method
res_file = 'result/bs_gen_%s.json' % method
score_file = 'result/bs_vqa_scores_%s.mat' % method
# Get model
model_fn = get_model_creation_fn('VAQ-Var')
create_fn = create_reader('VAQ-VVIS', phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# get data reader
subset = 'kptest'
reader = create_fn(batch_size=1, subset=subset, version=FLAGS.test_version)
exemplar = ExemplarLanguageModel()
if checkpoint_path is None:
if FLAGS.checkpoint_dir:
ckpt_dir = FLAGS.checkpoint_dir
else:
ckpt_dir = FLAGS.checkpoint_pat % (FLAGS.version, FLAGS.model_type)
# ckpt_dir = '/import/vision-ephemeral/fl302/models/v2_kpvaq_VAQ-RL/'
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
checkpoint_path = ckpt.model_checkpoint_path
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.ex
model = model_fn(model_config, 'sampling')
model.set_num_sampling_points(1000)
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
# build language model
language_model = LanguageModel()
language_model.build()
language_model.set_cache_dir('test_empty')
# language_model.set_cache_dir('v1_var_att_lowthresh_cache_restval_VAQ-VarRL')
language_model.set_session(sess)
language_model.setup_model()
# build VQA model
vqa_model = VQAWrapper(g, sess)
# vqa_model = MLBWrapper()
num_batches = reader.num_batches
print('Running beam search inference...')
results = []
batch_vqa_scores = []
num = FLAGS.max_iters if FLAGS.max_iters > 0 else num_batches
for i in range(num):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
im, _, _, top_ans, ans_tokens, ans_len = outputs[:-2]
# pdb.set_trace()
if top_ans == 2000:
continue
print('\n%d/%d' % (i, num))
question_id = int(quest_ids[0])
image_id = int(image_ids[0])
t1 = time()
pathes, scores = model.greedy_inference([im, ans_tokens, ans_len],
sess)
# find unique
ivqa_scores, ivqa_pathes = process_one(scores, pathes)
t2 = time()
print('Time for sample generation: %0.2fs' % (t2 - t1))
# apply language model
language_model_inputs = wrap_samples_for_language_model(
[ivqa_pathes], pad_token=model.pad_token - 1, max_length=20)
match_gt = exemplar.query(ivqa_pathes)
legality_scores = language_model.inference(language_model_inputs)
legality_scores[match_gt] = 1.0
num_keep = max(100, (legality_scores > 0.1).sum()) # no less than 100
valid_inds = (-legality_scores).argsort()[:num_keep]
t3 = time()
print('Time for language model filtration: %0.2fs' % (t3 - t2))
# for idx in valid_inds:
# path = ivqa_pathes[idx]
# sc = legality_scores[idx]
# sentence = to_sentence.index_to_question(path)
# # questions.append(sentence)
# print('%s (%0.3f)' % (sentence, sc))
# apply VQA model
sampled = [ivqa_pathes[_idx] for _idx in valid_inds]
# vqa_scores = vqa_model.get_scores(sampled, image_id, top_ans)
vqa_scores, is_valid = vqa_model.get_scores(sampled, im, top_ans)
# conf_inds = (-vqa_scores).argsort()[:20]
conf_inds = np.where(is_valid)[0]
# pdb.set_trace()
# conf_inds = (-vqa_scores).argsort()[:40]
t4 = time()
print('Time for VQA verification: %0.2fs' % (t4 - t3))
this_mean_vqa_score = vqa_scores[conf_inds].mean()
print('sampled: %d, unique: %d, legal: %d, gt: %d, mean score %0.2f' %
(pathes.shape[0], len(ivqa_pathes), num_keep, match_gt.sum(),
this_mean_vqa_score))
batch_vqa_scores.append(this_mean_vqa_score)
for _pid, idx in enumerate(conf_inds):
path = sampled[idx]
sc = vqa_scores[idx]
sentence = to_sentence.index_to_question(path)
aug_quest_id = question_id * 1000 + _pid
res_i = {
'image_id': int(image_id),
'question_id': aug_quest_id,
'question': sentence,
'score': float(sc)
}
results.append(res_i)
save_json(res_file, results)
batch_vqa_scores = np.array(batch_vqa_scores, dtype=np.float32)
mean_vqa_score = batch_vqa_scores.mean()
from scipy.io import savemat
savemat(score_file, {
'scores': batch_vqa_scores,
'mean_score': mean_vqa_score
})
print('BS mean VQA score: %0.3f' % mean_vqa_score)
return res_file, mean_vqa_score
def train():
model_config = ModelConfig()
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
reader_fn = create_reader('VAQ-Epoch', phase='train')
env = MixReward()
env.diversity_reward.mode = 'winner_take_all'
env.set_language_thresh(0.1)
env.set_replay_buffer(
insert_thresh=0.1,
sv_dir='vqa_replay_buffer/adv') # if 0.5, already fooled others
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.version, FLAGS.model_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
ckpt_suffix = train_dir.split('/')[-1]
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate *
0.1)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor,
staircase=False)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Setup summaries
summary_op = tf.summary.merge_all()
# Setup language model
lm = LanguageModel()
lm.build()
lm.set_cache_dir(ckpt_suffix)
env.set_language_model(lm)
# Setup VQA model
vqa = VQAModel(vqa_config.ModelConfig(), phase='train')
vqa.build()
env.set_vqa_model(vqa)
# create reader
reader = reader_fn(
batch_size=16,
subset='kprestval', # 'kptrain'
version=FLAGS.version)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
model=model,
summary_op=summary_op,
env=env)
def ivqa_decoding_beam_search(checkpoint_path=None):
model_config = ModelConfig()
method = FLAGS.method
res_file = 'result/bs_cand_for_vis.json'
# Get model
model_fn = get_model_creation_fn('VAQ-Var')
create_fn = create_reader('VAQ-VVIS', phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval',
top_ans_file='../VQA-tensorflow/data/vqa_trainval_top2000_answers.txt')
# get data reader
subset = 'kpval'
reader = create_fn(batch_size=1, subset=subset,
version=FLAGS.test_version)
exemplar = ExemplarLanguageModel()
if checkpoint_path is None:
if FLAGS.checkpoint_dir:
ckpt_dir = FLAGS.checkpoint_dir
else:
ckpt_dir = FLAGS.checkpoint_pat % (FLAGS.version, FLAGS.model_type)
# ckpt_dir = '/import/vision-ephemeral/fl302/models/v2_kpvaq_VAQ-RL/'
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
checkpoint_path = ckpt.model_checkpoint_path
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.ex
model = model_fn(model_config, 'sampling')
model.set_num_sampling_points(5000)
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
# build language model
language_model = LanguageModel()
language_model.build()
language_model.set_cache_dir('test_empty')
# language_model.set_cache_dir('v1_var_att_lowthresh_cache_restval_VAQ-VarRL')
language_model.set_session(sess)
language_model.setup_model()
# build VQA model
# vqa_model = N2MNWrapper()
# vqa_model = MLBWrapper()
num_batches = reader.num_batches
quest_ids_to_vis = {5682052: 'bread',
965492: 'plane',
681282: 'station'}
print('Running beam search inference...')
results = []
batch_vqa_scores = []
num = FLAGS.max_iters if FLAGS.max_iters > 0 else num_batches
for i in range(num):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
quest_id_key = int(quest_ids)
if quest_id_key not in quest_ids_to_vis:
continue
# pdb.set_trace()
im, gt_q, _, top_ans, ans_tokens, ans_len = outputs[:-2]
# pdb.set_trace()
if top_ans == 2000:
continue
print('\n%d/%d' % (i, num))
question_id = int(quest_ids[0])
image_id = int(image_ids[0])
t1 = time()
pathes, scores = model.greedy_inference([im, ans_tokens, ans_len], sess)
# find unique
ivqa_scores, ivqa_pathes = process_one(scores, pathes)
t2 = time()
print('Time for sample generation: %0.2fs' % (t2 - t1))
# apply language model
language_model_inputs = wrap_samples_for_language_model([ivqa_pathes],
pad_token=model.pad_token - 1,
max_length=20)
match_gt = exemplar.query(ivqa_pathes)
legality_scores = language_model.inference(language_model_inputs)
legality_scores[match_gt] = 1.0
num_keep = max(100, (legality_scores > 0.1).sum()) # no less than 100
valid_inds = (-legality_scores).argsort()[:num_keep]
print('keep: %d/%d' % (num_keep, len(ivqa_pathes)))
t3 = time()
print('Time for language model filtration: %0.2fs' % (t3 - t2))
def token_arr_to_list(arr):
return arr.flatten().tolist()
for _pid, idx in enumerate(valid_inds):
path = ivqa_pathes[idx]
# sc = vqa_scores[idx]
sentence = to_sentence.index_to_question(path)
aug_quest_id = question_id * 1000 + _pid
res_i = {'image_id': int(image_id),
'aug_id': aug_quest_id,
'question_id': question_id,
'target': sentence,
'top_ans_id': int(top_ans),
'question': to_sentence.index_to_question(token_arr_to_list(gt_q)),
'answer': to_sentence.index_to_answer(token_arr_to_list(ans_tokens))}
results.append(res_i)
save_json(res_file, results)
return None
def ivqa_decoding_beam_search(checkpoint_path=None):
model_config = ModelConfig()
method = FLAGS.method
res_file = 'result/bs_gen_%s.json' % method
score_file = 'result/bs_vqa_scores_%s.mat' % method
# Get model
model_fn = get_model_creation_fn('VAQ-Var')
create_fn = create_reader('VAQ-VVIS', phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# get data reader
subset = 'kptrain'
reader = create_fn(batch_size=1, subset=subset, version=FLAGS.test_version)
exemplar = ExemplarLanguageModel()
if checkpoint_path is None:
if FLAGS.checkpoint_dir:
ckpt_dir = FLAGS.checkpoint_dir
else:
ckpt_dir = FLAGS.checkpoint_pat % (FLAGS.version, FLAGS.model_type)
# ckpt_dir = '/import/vision-ephemeral/fl302/models/v2_kpvaq_VAQ-RL/'
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
checkpoint_path = ckpt.model_checkpoint_path
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.ex
model = model_fn(model_config, 'sampling')
model.set_num_sampling_points(5)
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
# build language model
language_model = LanguageModel()
language_model.build()
language_model.set_cache_dir('test_empty')
# language_model.set_cache_dir('v1_var_att_lowthresh_cache_restval_VAQ-VarRL')
language_model.set_session(sess)
language_model.setup_model()
num_batches = reader.num_batches
print('Running beam search inference...')
num = FLAGS.max_iters if FLAGS.max_iters > 0 else num_batches
neg_pathes = []
need_stop = False
for i in range(num):
outputs = reader.get_test_batch()
# inference
im, _, _, top_ans, ans_tokens, ans_len = outputs[:-2]
if top_ans == 2000:
continue
print('\n%d/%d' % (i, num))
t1 = time()
pathes, scores = model.greedy_inference([im, ans_tokens, ans_len],
sess)
# find unique
ivqa_scores, ivqa_pathes = process_one(scores, pathes)
t2 = time()
print('Time for sample generation: %0.2fs' % (t2 - t1))
# apply language model
language_model_inputs = wrap_samples_for_language_model(
[ivqa_pathes], pad_token=model.pad_token - 1, max_length=20)
match_gt = exemplar.query(ivqa_pathes)
legality_scores = language_model.inference(language_model_inputs)
legality_scores[match_gt] = 1.0
neg_inds = np.where(legality_scores < 0.2)[0]
for idx in neg_inds:
ser_neg = serialize_path(ivqa_pathes[idx][1:])
neg_pathes.append(ser_neg)
if len(neg_pathes) > 100000:
need_stop = True
break
# if len(neg_pathes) > 1000:
# need_stop = True
# break
# print('Neg size: %d' % len(neg_pathes))
if need_stop:
break
sv_file = 'data/lm_init_neg_pathes.json'
save_json(sv_file, neg_pathes)
def ivqa_decoding_beam_search(ckpt_dir, method):
model_config = ModelConfig()
inf_type = 'beam'
assert (inf_type in ['beam', 'rand'])
# method = FLAGS.method
if inf_type == 'rand':
res_file = 'result/bs_RL2_cands_LM_%s.json' % method
else:
res_file = 'result/bs_RL2_cands_LM_%s_BEAM.json' % method
if os.path.exists(res_file):
print('File %s already exist, skipped' % res_file)
return
# score_file = 'result/bs_vqa_scores_%s.mat' % method
# Get model
model_fn = get_model_creation_fn('VAQ-Var')
create_fn = create_reader('VAQ-VVIS', phase='test')
# Create the vocabulary.
to_sentence = SentenceGenerator(trainset='trainval')
# get data reader
subset = 'bs_test'
reader = create_fn(batch_size=1, subset=subset,
version=FLAGS.test_version)
exemplar = ExemplarLanguageModel()
# if checkpoint_path is None:
# if FLAGS.checkpoint_dir:
# ckpt_dir = FLAGS.checkpoint_dir
# else:
# ckpt_dir = FLAGS.checkpoint_pat % (FLAGS.version, FLAGS.model_type)
# ckpt_dir = '/import/vision-ephemeral/fl302/models/v2_kpvaq_VAQ-RL/'
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
checkpoint_path = ckpt.model_checkpoint_path
# Build model
g = tf.Graph()
with g.as_default():
# Build the model.ex
if inf_type == 'rand':
model = model_fn(model_config, 'sampling')
model.set_num_sampling_points(1000)
else:
model = model_fn(model_config, 'sampling_beam')
model.set_num_sampling_points(1000)
model.build()
# Restore from checkpoint
restorer = Restorer(g)
sess = tf.Session()
restorer.restore(sess, checkpoint_path)
# build language model
language_model = LanguageModel()
language_model.build()
language_model.set_cache_dir('test_empty')
# language_model.set_cache_dir('v1_var_att_lowthresh_cache_restval_VAQ-VarRL')
language_model.set_session(sess)
language_model.setup_model()
# build VQA model
# vqa_model = N2MNWrapper()
# vqa_model = MLBWrapper()
num_batches = reader.num_batches
print('Running beam search inference...')
results = {}
# batch_vqa_scores = []
num = FLAGS.max_iters if FLAGS.max_iters > 0 else num_batches
for i in range(num):
outputs = reader.get_test_batch()
# inference
quest_ids, image_ids = outputs[-2:]
im, _, _, top_ans, ans_tokens, ans_len = outputs[:-2]
# pdb.set_trace()
if top_ans == 2000:
continue
print('\n%d/%d' % (i, num))
question_id = int(quest_ids[0])
image_id = int(image_ids[0])
t1 = time()
pathes, scores = model.greedy_inference([im, ans_tokens, ans_len], sess)
# find unique
ivqa_scores, ivqa_pathes = process_one(scores, pathes)
t2 = time()
print('Time for sample generation: %0.2fs' % (t2 - t1))
# apply language model
language_model_inputs = wrap_samples_for_language_model([ivqa_pathes],
pad_token=model.pad_token - 1,
max_length=20)
match_gt = exemplar.query(ivqa_pathes)
legality_scores = language_model.inference(language_model_inputs)
legality_scores[match_gt] = 1.0
num_keep = max(100, (legality_scores > 0.3).sum()) # no less than 100
valid_inds = (-legality_scores).argsort()[:num_keep]
t3 = time()
print('Time for language model filtration: %0.2fs' % (t3 - t2))
# for idx in valid_inds:
# path = ivqa_pathes[idx]
# sc = legality_scores[idx]
# sentence = to_sentence.index_to_question(path)
# # questions.append(sentence)
# print('%s (%0.3f)' % (sentence, sc))
# apply VQA model
sampled = [ivqa_pathes[_idx] for _idx in valid_inds]
legality_scores = legality_scores[valid_inds]
result_key = int(question_id)
tmp = []
for idx, path in enumerate(sampled):
# path = sampled[idx]
sc = legality_scores[idx]
sentence = to_sentence.index_to_question(path)
# aug_quest_id = question_id * 1000 + _pid
res_i = {'image_id': int(image_id),
'aug_id': idx,
'question_id': question_id,
'question': sentence,
'score': float(sc)}
tmp.append(res_i)
print('Number of unique questions: %d' % len(tmp))
results[result_key] = tmp
save_json(res_file, results)