def predict(args_):
checkpoint_path = args_.checkpoint_path
words_file = args_.words_file
image_file = args_.path
if not os.path.exists(checkpoint_path):
print('checkpoint path is not exist.')
exit(0)
if not os.path.exists(words_file):
print('words file not found.')
exit(0)
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(), checkpoint_path)
g.finalize()
vocab = vocabulary.Vocabulary(words_file)
if os.path.isdir(image_file):
with tf.Session(graph=g) as sess:
restore_fn(sess)
generator = caption_generator.CaptionGenerator(model, vocab)
# sent a directory contains images
file_names = [os.path.join(image_file, i) for i in os.listdir(image_file) if i.lower().endswith('.jpg')
or i.lower().endswith('jpeg') or i.lower().endswith('png')]
file_names = [i for i in file_names if os.path.isfile(i)]
for f in file_names:
with tf.gfile.GFile(f, "rb") as img_file:
image = img_file.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(f))
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
# cv2 show image
image_array = cv2.imread(f, cv2.COLOR_BGR2RGB)
cv2.imshow('image', image_array)
cv2.waitKey(0)
elif os.path.isfile(image_file):
# sent a single image file
with tf.Session(graph=g) as sess:
restore_fn(sess)
generator = caption_generator.CaptionGenerator(model, vocab)
# sent a directory contains images
with tf.gfile.GFile(image_file, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(f))
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
else:
print('image path: {} not found.'.format(image_file))
exit(0)
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
prediction_dir = FLAGS.prediction_dir
if not tf.gfile.IsDirectory(prediction_dir):
tf.logging.info("Creating prediction directory: %s", prediction_dir)
tf.gfile.MakeDirs(prediction_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model = classifier_model.Classifier(model_config, mode="prediction")
model.build()
global MAX_NUM_TOKENS
MAX_NUM_TOKENS = model_config.sentence_length
init_op = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables())
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
start = time.time()
tf.logging.info("Starting prediction at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, init_op)
except KeyboardInterrupt:
pass
def model_fn(features, labels, mode, params):
im_mode = MODEKEY_TO_MODE[mode]
model_config = configuration.ModelConfig()
training_config = configuration.TrainingConfig()
model = show_and_tell_model.ShowAndTellModel(
model_config, mode=im_mode, train_inception=FLAGS.train_inception)
model.build_model_for_tpu(images=features["images"],
input_seqs=features["input_seqs"],
target_seqs=features["target_seqs"],
input_mask=features["input_mask"])
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=training_config.initial_learning_rate)
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
optimizer, training_config.clip_gradients)
if FLAGS.use_tpu:
optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
train_op = optimizer.minimize(
model.total_loss, global_step=tf.train.get_or_create_global_step())
def scaffold_fn():
"""Load pretrained Inception checkpoint at initialization time."""
return tf.train.Scaffold(init_fn=model.init_fn)
return tf.contrib.tpu.TPUEstimatorSpec(mode=mode,
loss=model.total_loss,
train_op=train_op,
scaffold_fn=scaffold_fn)
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
extraction_dir = FLAGS.extraction_dir
if not tf.gfile.IsDirectory(extraction_dir):
tf.logging.info("Creating extraction directory: %s", extraction_dir)
tf.gfile.MakeDirs(extraction_dir)
# generate eval dump file
dump_file = open(os.path.join(extraction_dir, FLAGS.extraction_file + '.json'), 'w')
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = classifier_model.Classifier(model_config, mode="extract")
model.build()
init_op = tf.group(tf.initialize_all_variables(), tf.initialize_local_variables())
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
start = time.time()
tf.logging.info("Starting extraction at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, dump_file, init_op)
except KeyboardInterrupt:
dump_file.close()
def main(_):
# Build the inference graph.
g = tf.Graph()
model_path = '/Users/harshpyati/personal/fyp/text_gen/model.ckpt-2000000'
vocab_path = '/Users/harshpyati/personal/fyp/text_gen/word_counts.txt'
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
model_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(vocab_path)
all_files = FLAGS.input_files.split(',')
files = []
for fil in all_files:
word = None
if '[' in fil:
word = fil.replace('[', '')
if ']' in fil:
word = fil.replace(']', '')
if ' ' in fil:
word = fil.replace(' ', '')
if "u'" in fil:
word = fil.replace("u'", '')
if '\'' in fil:
word = fil.replace("'", '')
if "'" in fil:
word = fil.replace("'", '')
if "[u" in fil:
word = fil.replace("[u", '')
if " u" in fil:
word = fil.replace(" u", '')
word = word.split('\'')[1]
files.append(word)
filenames = []
with tf.Session(graph=g) as sess:
generator = caption_generator.CaptionGenerator(model, vocab)
# Load the model from checkpoint.
restore_fn(sess)
all_captions = []
for file_pattern in files:
filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), file_pattern)
with tf.gfile.GFile(file_pattern, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
for index, caption in enumerate(captions):
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
data = {
"name": file_pattern,
"caption": sentence
}
all_captions.append(data)
break
print(all_captions)
def run():
inference_dir = FLAGS.inference_dir
if not tf.gfile.IsDirectory(inference_dir):
tf.logging.info("Creating inference directory: %s", inference_dir)
tf.gfile.MakeDirs(inference_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = auto_encoder_model.Auto_Encoder_Model(model_config,
mode="inference")
model.build()
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(inference_dir)
# g.finalize()
while True:
start = time.time()
tf.logging.info(
"Starting val at " +
time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op, model_config)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
def main(_):
with open(FLAGS.keyword_pickle_file,'r')as f:
keyword_data=cPickle.load(f)
with open(FLAGS.test_json_path)as f:
test_json=json.load(f)
id_to_filename=test_json['images']
id_to_path=[{'path':os.path.join(FLAGS.image_path,x['file_name']),'id':x['id']}for x in id_to_filename]
result_json=[]
g=tf.Graph()
with g.as_default():
model=inference_wrapper.InferenceWrapper()
restore_fn=model.build_graph_from_config(configuration.ModelConfig(),FLAGS.checkpoint_path)
g.finalize()
vocab=vocabulary.Vocabulary(FLAGS.vocab_file)
with tf.Session(graph=g)as sess:
restore_fn(sess)
generator=caption_generator.CaptionGenerator(model,vocab)
for data in id_to_path:
filename=data['path']
with tf.gfile.GFile(filename,"r")as f:
image=f.read()
captions=generator.beam_search(sess,image,keyword_data[os.path.basename(filename)])
print("Captions for image %s:"%os.path.basename(filename))
result={'image_id':data['id'],'caption':(" ".join([vocab.id_to_word(w)for w in captions[0].sentence[1:-1]])).decode('utf-8')}
print(result)
result_json.append(result)
with open(os.path.join(FLAGS.temp_path,"result.json"),'w')as f:
json.dump(result_json,f)
coco=COCO(FLAGS.test_json_path)
cocoRes=coco.loadRes(os.path.join(FLAGS.temp_path,"result.json"))
cocoEval=COCOEvalCap(coco,cocoRes)
cocoEval.evaluate()
def run(data, checkpoint_dir, eval_interval_secs, min_global_step, num_eval_examples): """Runs evaluation in a loop. Args: data: a pointer to teh MNIST data checkpoint_dir: Directory containing model checkpoints. eval_interval_secs: Interval between consecutive evaluations. min_global_step: Number of steps until the first evaluation. num_eval_examples: Number of examples to run the evaluation on. """ g = tf.Graph() with g.as_default(): # Build the model for evaluation. model_config = configuration.ModelConfig() the_model = model.DAE(model_config) the_model.build() # Create the Saver to restore model Variables. saver = tf.train.Saver() g.finalize() # Run a new evaluation run every eval_interval_secs. while True: start = time.time() # Run evaluation. run_once(data, the_model, saver, checkpoint_dir, min_global_step, num_eval_examples) time_to_next_eval = start + eval_interval_secs - time.time() # Wait until the time to next evaluation elapses if time_to_next_eval > 0: time.sleep(time_to_next_eval)
def run():
eval_dir=FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s",eval_dir)
tf.gfile.MakeDirs(eval_dir)
g=tf.Graph()
with g.as_default():
model_config=configuration.ModelConfig()
model_config.input_file_pattern=FLAGS.input_file_pattern
model=show_and_tell_model.ShowAndTellModel(model_config,mode="eval")
model.build()
saver=tf.train.Saver()
summary_op=tf.summary.merge_all()
summary_writer=tf.summary.FileWriter(eval_dir)
g.finalize()
i=-1
while True:
start=time.time()
tf.logging.info("Starting evaluation at "+time.strftime("%Y-%m-%d-%H:%M:%S",time.localtime()))
current_filenames=os.listdir(FLAGS.checkpoint_dir)
nums=[]
for x in current_filenames:
if x[-5:]=='index':
nums.append(int(re.findall(r'\d+',x)[0]))
nums.sort()
for x in nums:
if x>i:
run_once(model,saver,os.path.join(FLAGS.checkpoint_dir,'model.ckpt-'+str(x)),summary_writer,summary_op)
i=x
break
time_to_next_eval=start+FLAGS.eval_interval_secs-time.time()
if time_to_next_eval>0:
time.sleep(time_to_next_eval)
def testCallModelFnWithPlaceholders(self):
with _reset_for_test() as session:
config = configuration.ModelConfig()
model = show_and_tell_model.ShowAndTellModel(config, mode='train')
def model_fn(images, input_seq, target_seq, input_mask):
model.build_model_for_tpu(images, input_seq, target_seq, input_mask)
return model.total_loss
images = tf.placeholder(tf.float32, shape=(1, 224, 224, 3))
input_seq = tf.placeholder(tf.int32, shape=(1, 128))
target_seq = tf.placeholder(tf.int32, shape=(1, 128))
input_mask = tf.placeholder(tf.int32, shape=(1, 128))
tpu_model_fn = tpu.rewrite(model_fn,
[images, input_seq, target_seq, input_mask])
caption = np.random.randint(low=0, high=1000, size=128).reshape((1, 128))
session.run(tpu.initialize_system())
session.run(tf.global_variables_initializer())
inputs = {
images: np.random.randn(1, 224, 224, 3),
input_seq: caption,
target_seq: caption,
input_mask: np.random.random_integers(0, 1, size=128).reshape(1, 128),
}
session.run(tpu_model_fn, inputs)
session.run(tpu.shutdown_system())
def main(unused_argv):
assert FLAGS.input_file_pattern,"--input_file_pattern is required"
assert FLAGS.train_dir,"--train_dir is required"
model_config=configuration.ModelConfig()
model_config.input_file_pattern=FLAGS.input_file_pattern
model_config.inception_checkpoint_file=FLAGS.inception_checkpoint_file
training_config=configuration.TrainingConfig()
train_dir=FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s",train_dir)
tf.gfile.MakeDirs(train_dir)
g=tf.Graph()
with g.as_default():
model=show_and_tell_model.ShowAndTellModel(model_config,mode="train",train_inception=FLAGS.train_inception)
model.build()
learning_rate_decay_fn=None
if FLAGS.train_inception:
learning_rate=tf.constant(training_config.train_inception_learning_rate)
else:
learning_rate=tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor>0:
num_batches_per_epoch=(training_config.num_examples_per_epoch/model_config.batch_size)
decay_steps=int(num_batches_per_epoch*training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate,global_step):
return tf.train.exponential_decay(learning_rate,global_step,decay_steps=decay_steps,decay_rate=training_config.learning_rate_decay_factor,staircase=True)
learning_rate_decay_fn=_learning_rate_decay_fn
train_op=tf.contrib.layers.optimize_loss(loss=model.total_loss,global_step=model.global_step,learning_rate=learning_rate,optimizer=training_config.optimizer,clip_gradients=training_config.clip_gradients,learning_rate_decay_fn=learning_rate_decay_fn)
saver=tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep)
tf.contrib.slim.learning.train(train_op,train_dir,log_every_n_steps=FLAGS.log_every_n_steps,graph=g,global_step=model.global_step,number_of_steps=FLAGS.number_of_steps,init_fn=model.init_fn,saver=saver)
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
eval_dir = FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s", eval_dir)
tf.gfile.MakeDirs(eval_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = show_and_tell_model.ShowAndTellModel(model_config, mode="eval")
model.build()
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
# Create the summary operation and the summary writer.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir)
g.finalize()
# Run a new evaluation run every eval_interval_secs.
while True:
start = time.time()
tf.logging.info("Starting evaluation at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
def inference():
# build the inference graph
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper(FLAGS.rnn_type)
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filename = path
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
with tf.gfile.GFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(filename))
global sentences
sentences = []
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
sentences.append(sentence)
print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
def main(unused_argv):
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = hierarchy_text_attention.Hierarchy_text_attention(model_config)
model.build()
# Set up the learning rate.
learning_rate_decay_fn = None
learning_rate = tf.constant(model_config.lr)
if model_config.lr_decay > 0:
num_batches_per_epoch = (model_config.num_examples_per_epoch /
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
model_config.max_decay_epoch)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=model_config.lr_decay,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=model_config.optimizer,
clip_gradients=model_config.max_grad_norm,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=model_config.max_checkpoints_to_keep)
# Run training.automatic initialize the threads
summary_op = tf.summary.merge_all()
save_summaries_secs = 10
summary_writer = tf.summary.FileWriter('./log_train')
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
saver=saver,
summary_op=summary_op,
save_summaries_secs=save_summaries_secs,
summary_writer=summary_writer)
def main(_):
# Build the inference graph.
top_k = 4 # Print the top_k accuracy.
true_pred = np.zeros(top_k)
# Load pre-computed image features.
with open(FLAGS.feature_file, "rb") as f:
test_data = pkl.load(f)
test_ids = test_data.keys()
test_feat = np.zeros(
(len(test_ids), len(test_data[test_ids[0]]["image_feat"])))
test_rnn_feat = np.zeros(
(len(test_ids), len(test_data[test_ids[0]]["image_rnn_feat"])))
for i, test_id in enumerate(test_ids):
# Image feature in visual-semantic embedding space.
test_feat[i] = test_data[test_id]["image_feat"]
# Image feature in the RNN space.
test_rnn_feat[i] = test_data[test_id]["image_rnn_feat"]
g = tf.Graph()
with g.as_default():
model_config = configuration.ModelConfig()
model_config.rnn_type = FLAGS.rnn_type
model = polyvore_model.PolyvoreModel(model_config, mode="inference")
model.build()
saver = tf.train.Saver()
g.finalize()
with tf.Session() as sess:
saver.restore(sess, FLAGS.checkpoint_path)
questions = json.load(open(FLAGS.json_file))
all_pred = []
set_ids = []
all_scores = []
for question in questions:
score, pred = run_question_inference(
sess, question, test_ids, test_feat, test_rnn_feat,
model_config.num_lstm_units)
if pred != []:
all_pred.append(pred)
all_scores.append(score)
set_ids.append(question["question"][0].split("_")[0])
# 0 is the correct answer, iterate over top_k.
for i in range(top_k):
if 0 in pred[:i + 1]:
true_pred[i] += 1
# Print all top-k accuracy.
for i in range(top_k):
print("Top %d Accuracy: " % (i + 1))
print("%d correct answers in %d valid questions." %
(true_pred[i], len(all_pred)))
print("Accuracy: %f" % (true_pred[i] / len(all_pred)))
s = np.empty((len(all_scores), ), dtype=np.object)
for i in range(len(all_scores)):
s[i] = all_scores[i]
with open(FLAGS.result_file, "wb") as f:
pkl.dump({"set_ids": set_ids, "pred": all_pred, "score": s}, f)
def load_model(self):
print("Loading model with an input size of: [" +
str(self.input_width) + "," + str(self.input_height) + "]")
graph = tf.Graph()
with graph.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(
configuration.ModelConfig(),
os.path.join(self.model_dir,
"model.ckpt-" + str(self.checkpoint)))
graph.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(
os.path.join(self.model_dir, "word_counts.txt"))
sess = tf.Session(graph=graph)
restore_fn(sess)
generator = caption_generator.CaptionGenerator(model, vocab)
self._sess = sess
self._generator = generator
self._vocab = vocab
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
eval_dir = FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s", eval_dir)
tf.gfile.MakeDirs(eval_dir)
# We are going to dump all evaluation metrics at each step into a file to have
# it easily accessible for later evaluation.
dump_fn = os.path.join(eval_dir, 'evaluation.json')
# Check if the dumpfile already exists
# If so, we initialize the global variables MAX_SCORE and GS_MAX_SCORE at first
if os.path.exists(dump_fn):
global MAX_SCORE, GS_MAX_SCORE
with open(dump_fn, 'r') as f:
for line in f:
curr = json.loads(line)
score = scoreIteration(curr)
if score >= MAX_SCORE:
MAX_SCORE = score
GS_MAX_SCORE = curr['global_step']
# generate eval dump file
dump_file = open(dump_fn, 'a')
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = classifier_model.Classifier(model_config, mode="eval")
model.build()
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
# Create the summary operation and the summary writer.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir)
g.finalize()
# Run a new evaluation run every eval_interval_secs.
try:
while True:
start = time.time()
tf.logging.info(
"Starting evaluation at " +
time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op, dump_file)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time(
)
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
except KeyboardInterrupt:
dump_file.close()
def main(unused_argv):
assert FLAGS.checkpoint_dir, "--checkpoint_dir is required"
model_config = configuration.ModelConfig()
training_config = configuration.TrainingConfig()
model_config.batch_size = 612
# Build The tf Graph
g = tf.Graph()
with g.as_default():
# Build the model,care about BN and scope-prefix
with tf.variable_scope("train"):
model = sim_model.SimModel(model_config, mode="inference")
model.build()
# Set up the Saver
restore = tf.train.Saver()
with g.as_default():
init = tf.global_variables_initializer()
# start
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True))
sess.run(init)
##Restore
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
restore.restore(sess, ckpt.model_checkpoint_path)
print('Successfully loaded model from %s' %
ckpt.model_checkpoint_path)
else:
print('No checkpoint file found at %s' % FLAGS.checkpoint_dir)
return
#TODO care about batch_size
preds_all = []
for feats in tqdm(reader.batch_inputs()):
start_time = time.time()
feed_dict = {
model.input_seqs: feats[0],
model.input_mask: feats[1],
model.labels: feats[2],
}
score_value = sess.run(model.preds, feed_dict)
preds_all.append(np.squeeze(score_value))
# generate submit
test_ids = np.load('data/test/test_ids.npy')
preds = np.hstack(preds_all) # * .75
assert len(test_ids) == len(preds)
submission = pd.DataFrame({
'is_duplicate': preds.ravel(),
'test_id': test_ids
})
submission.to_csv('submit_logs/' + FLAGS.submit_name + '.csv',
index=False)
print("done!")
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
# q&a: understand follow snippets
# filenames = []
# for file_pattern in FLAGS.input_files.split(","):
# # tf.gfile.Glob(pattern) Returns a list of files that match the given pattern(s)
# filenames.extend(tf.gfile.Glob(file_pattern))
# note: assert FLAGS.input_files == 'utils/test_file_abspath_flickr8k'
with open(FLAGS.input_files, 'r') as f:
filenames = f.readlines()
filenames = [filename.strip() for filename in filenames]
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
session_config = tf.ConfigProto()
session_config.gpu_options.allow_growth = True
with tf.Session(graph=g, config=session_config) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
json_file = list()
for count, filename in enumerate(filenames):
with tf.gfile.GFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess,
image) # 返回的是beam_size个caption
# print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
img_caption_dict = {}
img_caption_dict['filename'] = os.path.basename(filename)
# Ignore begin and end words.
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
img_caption_dict['caption'] = sentence
json_file.append(img_caption_dict)
if count % 50 == 0:
print("counter: %d" % count)
store_json_file("im2txt_flickr8k_cap_google.json", json_file)
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
for file_pattern in FLAGS.input_files.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
image_id_caption = []
j = 0
for filename in filenames:
with tf.gfile.GFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
j += 1
print(j)
print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end words.
# print(caption.sentence[1:-1])
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
# print(sentence)
sentence = "".join(sentence)
print(" %d) %s (p=%f)" %
(i, sentence, math.exp(caption.logprob)))
if not i:
image_id_caption.append(
{
"image_id": filename.split('/')[-1].replace(
".jpg", ""),
"caption": sentence
}, )
image_id_caption = json.dumps(image_id_caption).encode('utf-8')
data = json.loads(image_id_caption)
with open(FLAGS.captions_file, 'w') as f:
json.dump(data, f)
print("Saving captions file to path %s" % FLAGS.captions_file)
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
dirs = os.walk(FLAGS.image_dir)
for a, _, filelist in dirs:
for filename in filelist:
origin_name = a + filename
if origin_name.endswith('.jpg'):
filenames.append(origin_name)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
res = []
num = 1
for filename in filenames:
imgid_sentence = {}
with tf.gfile.GFile(filename, "r") as f:
image = f.read()
captions = generator.beam_search(sess, image)
# print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = "".join(sentence)
if i == 0:
if num % 100 ==0 :
print("Captions for image %s:" % os.path.basename(filename))
print("%d) %s (p=%f)" % (num,sentence, math.exp(caption.logprob)))
imgid_sentence['image_id'] = os.path.basename(filename).split('.')[0]
imgid_sentence['caption'] = sentence
res.append(imgid_sentence)
num = num + 1
with io.open(FLAGS.out_predict_json, 'w', encoding='utf-8') as fd:
fd.write(unicode(json.dumps(res,
ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': '))))
assert len(filenames) == len(res)
print("Finished process %d images!"%len(filenames))
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
infer = inference_wrapper.InferenceWrapper()
restore_fn = infer.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Load file in the provied directory
filenames = []
for file_pattern in FLAGS.input_files.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running text detection on %d files matching %s",
len(filenames), FLAGS.input_files)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
filenames.sort()
# Predict
for filename in filenames:
with tf.gfile.GFile(filename, "r") as f:
# Read image
cv_img = cv2.imread(filename)
image = f.read()
# Make prediction
tic = time.time()
text_bboxes = infer.inference_step(sess, image)
toc = time.time()
print("Prediction for image %s in %.3f ms" %
(os.path.basename(filename), (toc - tic) * 1000))
# Show the result
for i in range(len(text_bboxes)):
text = "{}: {:.3f}".format(i, float(text_bboxes[i][4]))
cv2.putText(cv_img, text, (int(text_bboxes[i][0]) + 5, int(text_bboxes[i][1]) + 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 2)
cv2.rectangle(cv_img,
(int(text_bboxes[i][0]), int(text_bboxes[i][1])),
(int(text_bboxes[i][2]), int(text_bboxes[i][3])),
(0,0,255), 2)
cv2.namedWindow('image', cv2.WND_PROP_FULLSCREEN)
cv2.resizeWindow('image', 1500, 900);
cv2.imshow('image', cv_img)
k = cv2.waitKey(0)
if k == ord('n'):
cv2.destroyAllWindows()
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
#for file_pattern in FLAGS.input_files.split(","):
# filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
config_sess = tf.ConfigProto()
config_sess.gpu_options.allow_growth = True
with tf.Session(graph=g, config=config_sess) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
test_path = r'C:\Users\PSIML-1.PSIML-1\Desktop\projekti\Image-Captioning\test_data'
filenames = os.listdir(test_path)
#captions_index = preprocess_captions()
j = 0
for filename in filenames:
full_fname = os.path.join(test_path, filename)
with tf.gfile.GFile(full_fname, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
best_captions = []
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
best_captions.append(" %d) %s\n" % (i, sentence))
#image_idx = int(filename.split('.')[0].split('_')[2])
#true_captions = captions_index[image_idx]
plot_image(full_fname, None, best_captions, j)
j += 1
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
# print("FLAGS.input_files", FLAGS.input_files)
for file_pattern in FLAGS.input_files.split(","): # original might be right?
# for file_pattern in FLAGS.input_files.split(","):
print("HIIII", file_pattern)
# # filenames.extend(tf.gfile.Glob(file_pattern))
# filenames.extend(tf.gfile.Glob("*.jpg"))
# print("filenames list", filenames)
filenames.extend(tf.gfile.Glob(file_pattern))
print("filenames", filenames)
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
print("generator")
for filename in filenames:
# print("*" * 10)
# print("\n" * 5)
# print("FILENAME", filename)
# print("\n" * 5)
# print("*" * 10)
# with tf.gfile.GFile(filename, "r") as f:
with tf.gfile.GFile(filename, 'rb') as f:
# https://github.com/tensorflow/tensorflow/issues/11312
image = f.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
print("DONE :)")
def main(argv):
inputfile = ' '
outputfile = ' '
try:
opts, args = getopt.getopt(argv,"hi:o",["ifile=","ofile="])
except getopt.GetoptError:
print("input/output error ")
sys.exit(2)
for opt, arg in opts:
if opt =='-h':
print ('usage: python run_inference.py -i <inputfile> -o <outuptfile>')
sys.exit()
elif opt in ('-i','--input'):
inputfile = arg
elif opt in ('-o','--output'):
outputfile = arg
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
if inputfile == ' ':
for file_pattern in FLAGS.input_files.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
else:
for file_pattern in inputfile.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
with tf.Session(graph=g) as sess:
restore_fn(sess)
generator = caption_generator.CaptionGenerator(model, vocab)
for filename in filenames:
with tf.gfile.FastGFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
# print("Captions for image %s using NIC model:" % os.path.basename(filename))
prob = []
for i, caption in enumerate(captions):
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
prob.append(caption.logprob)
# In this case, only the one with the largetst logprob is left for futher operation
for caption in captions:
sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
if 'UNK' in sentence:# if luckily the model recognized the text information itself
final = sentence
break
if caption.logprob == max(prob):
final = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
final = ' '.join(final)
img = Image.open(FLAGS.input_files)
def input_fn(params):
model_config = configuration.ModelConfig()
model_config.input_file_pattern = params["input_file_pattern"]
model_config.batch_size = params["batch_size"]
model_config.mode = params["mode"]
model = show_and_tell_model.ShowAndTellModel(model_config, mode="train")
model.build_inputs()
return {
"images": model.images,
"input_seqs": model.input_seqs,
"target_seqs": model.target_seqs,
"input_mask": model.input_mask
}
def test_model():
"""Evaluate the LTS model."""
model_config = configuration.ModelConfig()
model_config.data = FLAGS.test_dataset_name
test_ims_path = model_config.data + '/' + model_config.data + '_test_ims.npy'
test_caps_path = model_config.data + '/' + model_config.data + '_test_caps.txt'
print ('Loading dataset ...')
test_caps = []
with open(test_caps_path) as f:
for line in f:
test_caps.append(line.strip())
def main(unused_argv):
#Load configuration
model_config = configuration.ModelConfig()
#Load vocabulary object
vocab = pickle.load(open(FLAGS.vocab_dir, 'rb'))
original_embedding = download_embedding()
chr_embedding = process_embedding(vocab, original_embedding, model_config)
pickle.dump(chr_embedding, open('chr_embedding.pkl', 'wb'))
def model_predict(img_path):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
checkpoint_path = os.path.join(
os.path.dirname(os.path.abspath('__file__')), 'models')
print(checkpoint_path)
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary("./data/word_counts.txt")
filenames = []
for file_pattern in img_path.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), img_path)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
preds = ''
out_data = []
for filename in filenames:
with tf.gfile.GFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
preds = str(i + 1) + ") " + sentence + "(p=" + str(
round(math.exp(caption.logprob), 6)) + ")"
out_data.append(preds)
print(" %d) %s (p=%f)" %
(i, sentence, math.exp(caption.logprob)))
out_json = json.dumps(out_data)
#print(out_json)
return out_json
def main(_):
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
filenames = []
for file_pattern in FLAGS.input_files.split(","):
filenames.extend(tf.gfile.Glob(file_pattern))
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
out = []
for filename in filenames:
with tf.gfile.FastGFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(sess, image)
print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end words.
# print(caption.sentence[1:-1])
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
print(" %d) %s (p=%f)" %
(i, sentence, math.exp(caption.logprob)))
if (i == 0):
out = sentence
img = Image.open(FLAGS.input_files)
plt.imshow(img)
plt.axis('off')
plt.title(str(out))
plt.show()
