def infer_test(model_name, output_probs, x):
BATCH_SIZE = 20000
with tf.Session().as_default() as sess:
ds, filename = dataset.test_features_dataset()
ds_iter = ds.batch(BATCH_SIZE).make_initializable_iterator()
sess.run(ds_iter.initializer,
feed_dict={filename: paths.TEST_TF_RECORDS})
tf.global_variables_initializer().run()
saver = tf.train.Saver()
lines = open(
os.path.join(paths.CHECKPOINTS_DIR,
model_name + '_latest')).read().split('\n')
last_checkpoint = [
l.split(':')[1].replace('"', '').strip() for l in lines
if 'model_checkpoint_path:' in l
][0]
saver.restore(sess, os.path.join(paths.CHECKPOINTS_DIR,
last_checkpoint))
_, one_hot_decoder = dataset.one_hot_label_encoder()
breeds = one_hot_decoder(np.identity(consts.CLASSES_COUNT))
agg_test_df = None
try:
while True:
test_batch = sess.run(ds_iter.get_next())
inception_output = test_batch['inception_output']
ids = test_batch['id']
pred_probs = sess.run(output_probs,
feed_dict={x: inception_output.T})
#print(pred_probs.shape)
test_df = pd.DataFrame(data=pred_probs.T, columns=breeds)
test_df.index = ids
if agg_test_df is None:
agg_test_df = test_df
else:
agg_test_df = agg_test_df.append(test_df)
except tf.errors.OutOfRangeError:
print('End of the dataset')
agg_test_df.to_csv(paths.TEST_PREDICTIONS,
index_label='id',
float_format='%.17f')
print('predictions saved to %s' % paths.TEST_PREDICTIONS)
def convert_bson_2_record(input_bson_filename, output_tfrecords_filename, n=None, inception_feature=False):
one_hot_encoder, _ = dataset.one_hot_label_encoder(csv_path="data/category_names.csv")
# inception_graph = tf.Graph()
# inception_sess = tf.Session(graph=inception_graph)
#
# with inception_graph.as_default(), inception_sess.as_default():
# inception_model = inception.inception_model()
z = 0
data = bson.decode_file_iter(open(input_bson_filename, 'rb'))
opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
# def get_inception_ouput(img):
# with inception_graph.as_default():
# inception_output = inception_model(inception_sess, img).reshape(-1).tolist()
# return inception_output
with tf.python_io.TFRecordWriter(output_tfrecords_filename, options=opts) as writer:
for c, d in tqdm(enumerate(data), total=n):
n_img = len(d['imgs'])
for index in range(n_img):
img_raw = d['imgs'][index]['picture']
# img = np.array(imread(io.BytesIO(img_raw)))
# height = img.shape[0]
# width = img.shape[1]
product_id = d['_id']
_feature = {
'_id': int64_feature(product_id),
consts.IMAGE_RAW_FIELD: bytes_feature(img_raw)
}
# if inception_feature:
# inception_feature_ = get_inception_ouput(img_raw)
# _feature[consts.INCEPTION_OUTPUT_FIELD] = float_feature(inception_feature_)
if 'category_id' in d:
_feature[consts.LABEL_ONE_HOT_FIELD] = int64_feature(int(one_hot_encoder([str(d['category_id'])])[0]))
example = tf.train.Example(features=tf.train.Features(feature=_feature))
writer.write(example.SerializeToString())
z = z + 1
if (n is not None) and (z % n == 0):
print(z)
break
def infer(model_name, img_raw):
with tf.Graph().as_default(), tf.Session().as_default() as sess:
tensors = freeze.unfreeze_into_current_graph(
os.path.join(paths.FROZEN_MODELS_DIR, model_name + '.pb'),
tensor_names=[
consts.INCEPTION_INPUT_TENSOR, consts.OUTPUT_TENSOR_NAME
])
_, one_hot_decoder = dataset.one_hot_label_encoder()
probs = sess.run(
tensors[consts.OUTPUT_TENSOR_NAME],
feed_dict={tensors[consts.INCEPTION_INPUT_TENSOR]: img_raw})
breeds = one_hot_decoder(np.identity(consts.CLASSES_COUNT)).reshape(-1)
# print(breeds)
df = pd.DataFrame(data={'prob': probs.reshape(-1), 'breed': breeds})
return df.sort_values(['prob'], ascending=False)
def convert_train(tfrecords_path):
one_hot_encoder, _ = dataset.one_hot_label_encoder()
inception_graph = tf.Graph()
inception_sess = tf.Session(graph=inception_graph)
with inception_graph.as_default(), inception_sess.as_default():
incept_model = inception.inception_model()
with tf.Graph().as_default(), tf.Session().as_default() as sess:
labels_path = tf.placeholder(dtype=tf.string)
images_ds = tf.contrib.data.TextLineDataset(labels_path) \
.skip(1) \
.map(parse_row) \
.map(read_example)
labels_iter = images_ds.make_initializable_iterator()
next_label = labels_iter.get_next()
sess.run(labels_iter.initializer,
feed_dict={labels_path: paths.LABELS})
print('Writing ', tfrecords_path)
bar = pyprind.ProgBar(13000, update_interval=1, width=60)
augmenter = image_augmenter()
with tf.python_io.TFRecordWriter(
tfrecords_path,
tf.python_io.TFRecordCompressionType.NONE) as writer:
try:
while True:
id, img, breed_label = sess.run(next_label)
one_hot_label = one_hot_encoder([breed_label
]).reshape(-1).tolist()
def get_inception_ouput(img):
with inception_graph.as_default():
inception_output = incept_model(
inception_sess, img).reshape(-1).tolist()
return inception_output
# print(inception_output.shape)
# print('writing %s - %s' % (len(img), breed_label))
images = [img]
images.extend(augmenter(img))
for image in images:
example = build_train_example(
image, one_hot_label, breed_label,
get_inception_ouput(image))
writer.write(example.SerializeToString())
bar.update()
except tf.errors.OutOfRangeError:
print('End of the dataset')
writer.flush()
writer.close()
print('Finished')
def build_stanford_example(img_raw, inception_output, one_hot_label,
annotation):
example = tf.train.Example(features=tf.train.Features(
feature={
'label': bytes_feature(annotation['breed'].encode()),
consts.IMAGE_RAW_FIELD: bytes_feature(img_raw),
consts.LABEL_ONE_HOT_FIELD: float_feature(one_hot_label),
consts.INCEPTION_OUTPUT_FIELD: float_feature(inception_output)
}))
return example
if __name__ == '__main__':
one_hot_encoder, _ = dataset.one_hot_label_encoder()
with tf.Graph().as_default(), \
tf.Session().as_default() as sess, \
tf.python_io.TFRecordWriter(paths.STANFORD_DS_TF_RECORDS,
tf.python_io.TFRecordCompressionType.NONE) as writer:
incept_model = inception.inception_model()
def get_inception_ouput(img):
inception_output = incept_model(sess, img).reshape(-1).tolist()
return inception_output
for breed_dir in [d for d in os.listdir(annotations_root_dir)]:
print(breed_dir)
for annotation_file in [