def infer_test(model_name, output_probs, x):
BATCH_SIZE = 20000
with tf.Session().as_default() as sess:
ds, filename = dataset.test_features_dataset()
ds_iter = ds.batch(BATCH_SIZE).make_initializable_iterator()
sess.run(ds_iter.initializer,
feed_dict={filename: paths.TEST_TF_RECORDS})
tf.global_variables_initializer().run()
saver = tf.train.Saver()
lines = open(
os.path.join(paths.CHECKPOINTS_DIR,
model_name + '_latest')).read().split('\n')
last_checkpoint = [
l.split(':')[1].replace('"', '').strip() for l in lines
if 'model_checkpoint_path:' in l
][0]
saver.restore(sess, os.path.join(paths.CHECKPOINTS_DIR,
last_checkpoint))
_, one_hot_decoder = dataset.one_hot_label_encoder()
breeds = one_hot_decoder(np.identity(consts.CLASSES_COUNT))
agg_test_df = None
try:
while True:
test_batch = sess.run(ds_iter.get_next())
inception_output = test_batch['inception_output']
ids = test_batch['id']
pred_probs = sess.run(output_probs,
feed_dict={x: inception_output.T})
#print(pred_probs.shape)
test_df = pd.DataFrame(data=pred_probs.T, columns=breeds)
test_df.index = ids
if agg_test_df is None:
agg_test_df = test_df
else:
agg_test_df = agg_test_df.append(test_df)
except tf.errors.OutOfRangeError:
print('End of the dataset')
agg_test_df.to_csv(paths.TEST_PREDICTIONS,
index_label='id',
float_format='%.17f')
print('predictions saved to %s' % paths.TEST_PREDICTIONS)
def infer_test(model_name, x_, output_probs_, batch_size, test_tfrecords_files, test_prediction_csv):
_, one_hot_decoder = dataset.one_hot_label_encoder(csv_path="data/category_names.csv")
with tf.Session().as_default() as sess:
ds, filename = dataset.test_features_dataset()
ds_iter = ds.batch(batch_size).make_initializable_iterator()
sess.run(ds_iter.initializer, feed_dict={filename: test_tfrecords_files})
tf.global_variables_initializer().run()
saver = tf.train.Saver()
lines = open(os.path.join(paths.CHECKPOINTS_DIR, model_name, str(LEARNING_RATE), model_name + '_latest')).read().split('\n')
latest_checkpoint = [l.split(':')[1].replace('"', '').strip() for l in lines if 'model_checkpoint_path:' in l][0]
check_point_path = os.path.join(paths.CHECKPOINTS_DIR, model_name, str(LEARNING_RATE), latest_checkpoint)
print("Restore from {}".format(check_point_path))
saver.restore(sess, check_point_path)
with open(test_prediction_csv, 'w') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(["_id", "category_id", "prob"])
try:
while True:
test_batch = sess.run(ds_iter.get_next())
ids = test_batch['_id']
inception_features = test_batch[consts.INCEPTION_OUTPUT_FIELD]
print(ids.shape)
probabilities_ = sess.run(output_probs_, feed_dict={x_: inception_features})
pred_labels = one_hot_decoder(probabilities_)
max_probs = np.apply_along_axis(np.amax, 1, probabilities_)
for (_id, pred_label, prob) in zip(ids, pred_labels, max_probs):
csv_writer.writerow([_id, pred_label, prob])
# test_df = pd.DataFrame(data=zip(ids, pred_labels, max_probs), columns=["_id", "category_id", "prob"])
#
# idx = test_df.groupby(['_id'])['prob'].transform(max) == test_df['prob']
#
# test_df = test_df[idx]
#
# for index, row in test_df.iterrows():
# csv_writer.writerow([row._id, row.category_id, row.prob])
except tf.errors.OutOfRangeError:
print('End of the dataset')
# agg_test_df.to_csv(paths.TEST_PREDICTIONS, index_label='id', float_format='%.17f')
print('predictions saved to %s' % test_prediction_csv)
def infer_train(model_name, output_probs, x):
BATCH_SIZE = 20000
with tf.Session().as_default() as sess:
ds, filename = dataset.features_dataset()
ds_iter = ds.batch(BATCH_SIZE).make_initializable_iterator()
sess.run(ds_iter.initializer, feed_dict={filename: paths.TRAIN_TF_RECORDS})
tf.global_variables_initializer().run()
saver = tf.train.Saver()
lines = open(os.path.join(paths.CHECKPOINTS_DIR, model_name + '_latest')).read().split('\n')
last_checkpoint = [l.split(':')[1].replace('"', '').strip() for l in lines if 'model_checkpoint_path:' in l][0]
saver.restore(sess, os.path.join(paths.CHECKPOINTS_DIR, last_checkpoint))
_, one_hot_decoder = dataset.one_hot_label_encoder()
breeds = one_hot_decoder(np.identity(consts.CLASSES_COUNT))
agg_test_df = None
try:
while True:
test_batch = sess.run(ds_iter.get_next())
inception_output = test_batch['inception_output']
labels = test_batch['label']
#labels = [x[2:-1] for x in labels]
pred_probs = sess.run(output_probs, feed_dict={x: inception_output.T})
pred_probs_max = pred_probs >= np.max(pred_probs, axis=0)
pred_breeds = one_hot_decoder(pred_probs_max.T)
test_df = pd.DataFrame(data={'pred': pred_breeds, 'actual': labels})
if agg_test_df is None:
agg_test_df = test_df
else:
agg_test_df = agg_test_df.append(test_df)
except tf.errors.OutOfRangeError:
print('End of the dataset')
print(agg_test_df.take(range(0, 10)))
agg_test_df.to_csv(paths.TRAIN_CONFUSION, index_label='id', float_format='%.17f')
print('predictions saved to %s' % paths.TRAIN_CONFUSION)
def infer(model_name, img_raw):
with tf.Graph().as_default(), tf.Session().as_default() as sess:
tensors = freeze.unfreeze_into_current_graph(
os.path.join(paths.FROZEN_MODELS_DIR, model_name + '.pb'),
tensor_names=[consts.INCEPTION_INPUT_STRING_TENSOR, consts.OUTPUT_TENSOR_NAME])
_, one_hot_decoder = dataset.one_hot_label_encoder()
probs = sess.run(tensors[consts.OUTPUT_TENSOR_NAME],
feed_dict={tensors[consts.INCEPTION_INPUT_STRING_TENSOR]: img_raw})
breeds = one_hot_decoder(np.identity(consts.CLASSES_COUNT)).reshape(-1)
# print(breeds)
df = pd.DataFrame(data={'prob': probs.reshape(-1), 'breed': breeds})
return df.sort_values(['prob'], ascending=False)
return img_raw
def build_stanford_example(img_raw, inception_output, one_hot_label, annotation):
example = tf.train.Example(features=tf.train.Features(feature={
'label': bytes_feature(annotation['breed'].encode()),
consts.IMAGE_RAW_FIELD: bytes_feature(img_raw),
consts.LABEL_ONE_HOT_FIELD: float_feature(one_hot_label),
consts.INCEPTION_OUTPUT_FIELD: float_feature(inception_output)}))
return example
if __name__ == '__main__':
one_hot_encoder, _ = dataset.one_hot_label_encoder()
with tf.Graph().as_default(), \
tf.Session().as_default() as sess, \
tf.python_io.TFRecordWriter(paths.STANFORD_DS_TF_RECORDS,
tf.python_io.TFRecordCompressionType.NONE) as writer:
incept_model = inception.inception_model()
def get_inception_ouput(img):
inception_output = incept_model(sess, img).reshape(-1).tolist()
return inception_output
for breed_dir in [d for d in os.listdir(annotations_root_dir)]:
TEST_OUTPUT_CSV = args.csv_filename
else:
TEST_OUTPUT_CSV = cfg["TEST"]["OUTPUT_CSV_PATH"]
MODEL_NAME = cfg["MODEL"]["MODEL_NAME"]
MODEL_LAYERS = cfg["MODEL"]["MODEL_LAYERS"]
LEARNING_RATE = cfg["TRAIN"]["LEARNING_RATE"]
print("Testing model name: {}".format(MODEL_NAME))
print("Testing data: {}".format(TEST_TF_RECORDS))
print("Testing output: {}".format(TEST_OUTPUT_CSV))
batch_shape = [None, 180, 180, 3]
slim = tf.contrib.slim
_, one_hot_decoder = dataset.one_hot_label_encoder(csv_path="data/category_names.csv")
with tf.Graph().as_default():
x_input = tf.placeholder(tf.float32, shape=batch_shape)
with slim.arg_scope(inception.inception_v3_arg_scope()):
_, end_points = inception.inception_v3(x_input,
num_classes=5270,
is_training=True,
dropout_keep_prob=1.0)
variables_to_restore = slim.get_variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
predicted_labels = end_points['Predictions']
