def load_coco(self,
dataset_dir,
subset,
class_ids=None,
class_map=None,
return_coco=False):
"""Load a subset of the COCO dataset.
dataset_dir: The root directory of the COCO dataset.
subset: What to load (train, val, minival, val35k)
class_ids: If provided, only loads images that have the given classes.
class_map: TODO: Not implemented yet. Supports maping classes from
different datasets to the same class ID.
return_coco: If True, returns the COCO object.
"""
# Path
image_dir = os.path.join(
dataset_dir, "train2017" if subset == "train" else "val2017")
# Create COCO object
json_path_dict = {
"train": "annotations/instances_train2017.json",
"val": "annotations/instances_val2017.json"
# "minival": "annotations/instances_minival2017.json",
# "val35k": "annotations/instances_valminusminival2017.json",
}
coco = COCO(os.path.join(dataset_dir, json_path_dict[subset]))
# Load all classes or a subset?
if not class_ids:
# All classes
class_ids = sorted(coco.getCatIds())
# All images or a subset?
if class_ids:
image_ids = []
for id in class_ids:
image_ids.extend(list(coco.getImgIds(catIds=[id])))
# Remove duplicates
image_ids = list(set(image_ids))
else:
# All images
image_ids = list(coco.imgs.keys())
# Add classes
for i in class_ids:
self.add_class("coco", i, coco.loadCats(i)[0]["name"])
# Add images
for i in image_ids:
self.add_image("coco",
image_id=i,
path=os.path.join(image_dir,
coco.imgs[i]['file_name']),
width=coco.imgs[i]["width"],
height=coco.imgs[i]["height"],
annotations=coco.loadAnns(
coco.getAnnIds(imgIds=[i], iscrowd=False)))
if return_coco:
return coco
def __init__(self,
coco_name,
main_controller,
shuffle=True,
resize_dim=(1024, 1024)):
"""
COCO class is an adapter for coco dataset that ensures campatibility with ConvDet layer logic.
The dataset should be initialized with:
:param name: string with a name of the dataset. Good names can be 'train', 'test', or 'va', but I am not a stickler =)
:param path: a full path to the folder containing images and annotations folder
:param mc: main controller containing remaining parameters necessary for the proper initialization. mc should contain:
- mc.batch_size - an integer greater than 0
- mc.ANNOTATIONS_FILE_NAME - a file name located in the coco_path/annotations
- mc.BATCH_CLASSES - an array of classes to be learned (at least 1)
"""
self.shuffle = shuffle
self.imgIds = []
self.catIds = []
self.name = coco_name
IMDB.__init__(self,
resize_dim=resize_dim,
feature_map_size=main_controller.OUTPUT_RES,
main_controller=main_controller)
#1. Get an array of image indicies
assert type(main_controller.ANNOTATIONS_FILE_NAME)==str,\
"Provide a name of the file containing annotations in mc.ANNOTATIONS_FILE_NAME"
self.annotations_file = main_controller.ANNOTATIONS_FILE_NAME
self.coco = COCO(self.annotations_file)
categories = self.coco.loadCats(self.coco.getCatIds())
self.CLASS_NAMES_AVAILABLE = [
category['name'] for category in categories
]
self.CATEGORIES = set(
[category['supercategory'] for category in categories])
assert type(main_controller.BATCH_CLASSES) == list and len(main_controller.BATCH_CLASSES)>0,\
"Provide a list of classes to be learned in this batch through mc.BATCH_CLASSES"
self.BATCH_CLASSES = main_controller.BATCH_CLASSES
def main(train_type='Resnet'):
val_annFile = './annotations/instances_val2017.json'
val_coco = COCO(val_annFile)
val_set, val_len = prepare_dataset(val_coco, batch_size, [224, 224])
handle = tf.placeholder(tf.string, [])
iterator = tf.data.Iterator.from_string_handle(
handle,
(tf.float32, tf.float32, tf.float32),
((None, None, None, 3), (None, num_classes), (None, None, None, num_classes))
)
imgs, labels, gt = iterator.get_next()
val_iter = val_set.make_initializable_iterator()
val_handle = val_iter.string_handle()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
val_handle = sess.run(val_handle)
print('initial done')
tic = time.time()
if train_type == 'Resnet':
cnt = 0
total_loss = 0.0
sess.run(val_iter.initializer)
while True:
try:
img, y = sess.run([imgs, labels], feed_dict={handle: val_handle})
cnt += 16
print('total', cnt, img.shape, y.shape)
if cnt >= 128:
break
except tf.errors.OutOfRangeError:
break
else:
pass
print('Done (t={:0.2f}s)'.format(time.time()- tic))
import sys, os
# sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, './PythonAPI/')
# sys.path.insert(0, os.path.abspath('data'))
for _ in sys.path:
print(_)
from PythonAPI.pycocotools.coco import COCO
import cv2
import numpy as np
import os
from libs.label_name_dict import coco_dict
annotation_path = '/home/yjr/DataSet/COCO/2017/annotations/instances_train2017.json'
print("load coco .... it will cost about 17s..")
coco = COCO(annotation_path)
imgId_list = coco.getImgIds()
imgId_list = np.array(imgId_list)
total_imgs = len(imgId_list)
# print (NAME_LABEL_DICT)
def next_img(step):
if step % total_imgs == 0:
np.random.shuffle(imgId_list)
imgid = imgId_list[step % total_imgs]
def main(train_type='Resnet', restore=False, maxiter=10, test=False):
train_annFile = './annotations/instances_train2017.json'
val_annFile = './annotations/instances_val2017.json'
with open('PythonAPI/cat_dict.json', 'r') as f:
cat_dict = json.load(f)
train_coco = COCO(train_annFile)
val_coco = COCO(val_annFile)
train_key = []
val_key = []
for i in list(range(num_classes - 1)):
train_key.append(train_coco.cats[cat_dict['c2id'][str(i)]]['name'])
val_key.append(val_coco.cats[cat_dict['c2id'][str(i)]]['name'])
train_key.append('background')
val_key.append('background')
train_set, train_len = prepare_dataset(train_coco, batch_size, [224, 224])
val_set, val_len = prepare_dataset(val_coco, batch_size, [224, 224])
handle = tf.placeholder(tf.string, [])
iterator = tf.data.Iterator.from_string_handle(
handle, (tf.float32, tf.float32, tf.float32),
((None, None, None, 3), (None, num_classes),
(None, None, None, num_classes)))
imgs, labels, gt = iterator.get_next()
train_iter = train_set.make_initializable_iterator()
val_iter = val_set.make_initializable_iterator()
train_handle = train_iter.string_handle()
val_handle = val_iter.string_handle()
resnet_step = tf.Variable(0,
dtype=tf.int64,
name="resnet_step",
trainable=False)
deep_step = tf.Variable(0,
dtype=tf.int64,
name="deep_step",
trainable=False)
crf_step = tf.Variable(0, dtype=tf.int64, name="crf_step", trainable=False)
crf_seperate = tf.placeholder(tf.bool, shape=[])
_deeplab = deeplab.deeplab_v3_plus(imgs, [128, 64], [48, num_classes],
num_classes)
res_out = _deeplab.get_dense()
res_loss = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=tf.stop_gradient(labels), logits=res_out, name='res_loss')
res_mean_loss = tf.reduce_mean(res_loss)
res_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(
res_loss, global_step=resnet_step)
pred_out = _deeplab.get_pred()
pred_softmax = tf.nn.softmax(pred_out)
pred_loss = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=tf.stop_gradient(gt), logits=pred_out, name='pred_loss')
pred_mean_loss = tf.reduce_mean(pred_loss)
pred_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(
pred_loss, global_step=deep_step)
pred_acc = tf.reduce_mean(
tf.cast(tf.argmax(pred_out, -1) == tf.argmax(gt, -1), tf.float32))
crf_in = tf.cond(crf_seperate, lambda: tf.stop_gradient(pred_out),
lambda: pred_out)
crf_kernel = tf.constant([1., 1., 1., .5, .5], dtype=tf.float32)
crf_out = crf_rnn(crf_in, imgs, crf_kernel, 3, "crf_rnn")
crf_loss = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=tf.stop_gradient(gt), logits=crf_out, name='crf_loss')
crf_mean_loss = tf.reduce_mean(crf_loss)
crf_op = tf.train.AdamOptimizer(learning_rate=2e-5).minimize(
crf_loss, global_step=crf_step)
crf_acc = tf.reduce_mean(
tf.cast(tf.argmax(crf_out, -1) == tf.argmax(gt, -1), tf.float32))
crf_softmax = tf.nn.softmax(crf_out)
reader = pywrap_tensorflow.NewCheckpointReader(model_path)
restore_dict = dict()
for v in tf.get_collection(tf.GraphKeys.VARIABLES):
tname = v.name.split(':')[0]
if reader.has_tensor(tname):
restore_dict[tname] = v
saver = tf.train.Saver()
restorer = tf.train.Saver(restore_dict)
summary = summarizer(os.path.join(
log_dir,
'log%s.csv' % ('CRF' if train_type == 'CRF-only' else train_type)),
['step', 'train_loss', 'val_loss'],
25,
restore=restore)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
if restore:
sess.run(tf.global_variables_initializer())
restorer.restore(sess, model_path)
print('restored')
else:
sess.run(tf.global_variables_initializer())
print('initial done')
train_handle = sess.run(train_handle)
val_handle = sess.run(val_handle)
if test:
if train_type == "Deep":
pair = [imgs, pred_softmax]
elif train_type in ["CRF-only", "CRF"]:
pair = [imgs, crf_softmax]
else:
pair = [imgs, pred_softmax]
sess.run(val_iter.initializer)
cnt = 0
for epoc in range(100):
img, pred = sess.run(pair, feed_dict={handle: val_handle})
print('iter%d' % epoc)
for i in range(img.shape[0]):
plot.draw_raw_image(img[i][:, :, ::-1],
"./test/img_%d_raw.jpg" % cnt)
plot.draw_image(pred[i], "./test/img_%d_pred.jpg" % cnt)
cnt += 1
return
sess.run([train_iter.initializer, val_iter.initializer])
if train_type == 'Resnet':
cnt = 0
save_cnt = 0
epoc = 0
while epoc < maxiter:
_, _loss = sess.run([res_op, res_mean_loss],
feed_dict={handle: train_handle})
cnt += batch_size
save_cnt += 1
if summary.step == summary.steps - 1:
print("%d/%d %f" % (cnt, train_len, cnt / train_len))
_valloss = sess.run(res_mean_loss,
feed_dict={handle: val_handle})
summary.summary(train_loss=_loss,
val_loss=_valloss,
step=sess.run(resnet_step))
else:
summary.summary(train_loss=_loss)
if cnt >= train_len:
epoc += 1
print('epoc %d done' % epoc)
cnt -= train_len
saver.save(sess, model_path)
print('model saved')
save_cnt = 0
elif save_cnt >= 50:
save_cnt = 0
saver.save(sess, model_path)
print('model saved')
else:
pass
elif train_type == 'Deep':
cnt = 0
save_cnt = 0
epoc = 0
while epoc < maxiter:
_, _loss = sess.run([pred_op, pred_mean_loss],
feed_dict={handle: train_handle})
cnt += batch_size
save_cnt += 1
if summary.step == summary.steps - 1:
print("%d/%d %f" % (cnt, train_len, cnt / train_len))
_valloss = sess.run(pred_mean_loss,
feed_dict={handle: val_handle})
summary.summary(train_loss=_loss,
val_loss=_valloss,
step=sess.run(deep_step))
else:
summary.summary(train_loss=_loss)
if cnt >= train_len:
epoc += 1
print('epoc %d done' % epoc)
cnt -= train_len
saver.save(sess, model_path)
print('model saved')
save_cnt = 0
elif save_cnt >= 50:
save_cnt = 0
saver.save(sess, model_path)
print('model saved')
else:
pass
elif train_type in ["CRF-only", "CRF"]:
crf_only = (train_type == "CRF-only")
cnt = 0
save_cnt = 0
epoc = 0
while epoc < maxiter:
_, _loss = sess.run([crf_op, crf_mean_loss],
feed_dict={
handle: train_handle,
crf_seperate: crf_only
})
cnt += batch_size
save_cnt += 1
if summary.step == summary.steps - 1:
print("%d/%d %f" % (cnt, train_len, cnt / train_len))
_valloss = sess.run(crf_mean_loss,
feed_dict={
handle: val_handle,
crf_seperate: crf_only
})
summary.summary(train_loss=_loss,
val_loss=_valloss,
step=sess.run(crf_step))
else:
summary.summary(train_loss=_loss)
if cnt >= train_len:
epoc += 1
print('epoc %d done' % epoc)
cnt -= train_len
saver.save(sess, model_path)
print('model saved')
save_cnt = 0
elif save_cnt >= 50:
save_cnt = 0
saver.save(sess, model_path)
print('model saved')
else:
pass
else:
assert False, "unknown training type %s" % train_type
from PythonAPI.pycocotools.coco import COCO
import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
pylab.rcParams['figure.figsize'] = (8.0, 10.0)
dataDir='/home/home/AndroidStudioProjects/Messaging/visible/coco_dataset_images'
dataType='val2017'
annFile='{}/annotations/instances_{}.json'.format(dataDir,dataType)
# initialize COCO api for instance annotations
coco=COCO(annFile)
class coco(IMDB):
def __init__(self,
coco_name,
main_controller,
shuffle=True,
resize_dim=(1024, 1024)):
"""
COCO class is an adapter for coco dataset that ensures campatibility with ConvDet layer logic.
The dataset should be initialized with:
:param name: string with a name of the dataset. Good names can be 'train', 'test', or 'va', but I am not a stickler =)
:param path: a full path to the folder containing images and annotations folder
:param mc: main controller containing remaining parameters necessary for the proper initialization. mc should contain:
- mc.batch_size - an integer greater than 0
- mc.ANNOTATIONS_FILE_NAME - a file name located in the coco_path/annotations
- mc.BATCH_CLASSES - an array of classes to be learned (at least 1)
"""
self.shuffle = shuffle
self.imgIds = []
self.catIds = []
self.name = coco_name
IMDB.__init__(self,
resize_dim=resize_dim,
feature_map_size=main_controller.OUTPUT_RES,
main_controller=main_controller)
#1. Get an array of image indicies
assert type(main_controller.ANNOTATIONS_FILE_NAME)==str,\
"Provide a name of the file containing annotations in mc.ANNOTATIONS_FILE_NAME"
self.annotations_file = main_controller.ANNOTATIONS_FILE_NAME
self.coco = COCO(self.annotations_file)
categories = self.coco.loadCats(self.coco.getCatIds())
self.CLASS_NAMES_AVAILABLE = [
category['name'] for category in categories
]
self.CATEGORIES = set(
[category['supercategory'] for category in categories])
assert type(main_controller.BATCH_CLASSES) == list and len(main_controller.BATCH_CLASSES)>0,\
"Provide a list of classes to be learned in this batch through mc.BATCH_CLASSES"
self.BATCH_CLASSES = main_controller.BATCH_CLASSES
@property
def imgIds(self):
return self.__imgIds
@imgIds.setter
def imgIds(self, values):
assert type(values)==list,\
"imgIds is incorrect. Array is expected. {} was recieved.".format(type(values).__name__)
shuffle_flag = self.shuffle
if shuffle_flag:
shuffle(values)
self.__imgIds = values
@property
def annotations_file(self):
assert os.path.exists(self.__annotations_file),\
"Invalid path was provided for the data set. The following path doesn't exist: {}"\
.format(self.__annotations_file)
return self.__annotations_file
@annotations_file.setter
def annotations_file(self, value):
assert os.path.exists(value), "Annotations file doesn't exis at path {}. Please provide a full path to annotatinos." \
.format(value)
self.__annotations_file = value
BATCH_CLASSES = None
@property
def BATCH_CLASSES(self):
return self.__BATCH_CLASSES
@BATCH_CLASSES.setter
def BATCH_CLASSES(self, values):
for value in values:
assert value in self.CLASS_NAMES_AVAILABLE,\
"BATCH_CLASSES array is incorrect. The following class (from the array) " + \
"is not present in self.CLASS_NAMES_AVAILABLE array: {}.".\
format(value)
self.__BATCH_CLASSES = values
# build an array of file indicies
img_ids = []
cat_ids = []
for class_name in self.__BATCH_CLASSES:
cat_id = self.coco.getCatIds(catNms=[class_name])
cat_ids.extend(cat_id)
img_ids.extend(self.coco.getImgIds(catIds=cat_id))
# update image ids
self.imgIds = img_ids
self.catIds = cat_ids
def __provide_img_id(self, id):
return self.imgIds[id]
def epoch_size(self):
return len(self.imgIds)
def __provide_img_file_name(self, id):
"""
Protocol describing the implementation of a method that provides the name of the image file based on
an image id.
:param id: dataset specific image id
:return: string containing file name
"""
descriptions = self.coco.loadImgs(id)[0]
return descriptions['file_name']
def __provide_img_tags(self, id, coco_labels=True):
"""
Protocol describing the implementation of a method that provides tags for the image file based on
an image id.
:param id: dataset specific image id
:param coco_labels(optional): indicates wherher coco label ids should be returned
default value is False - BATCH_CLASS ids should be returned
:return: an array containing the list of tags
"""
# Extract annotation ids
ann_ids = self.coco.getAnnIds(imgIds=[id],
catIds=self.catIds,
iscrowd=None)
# get all annotations available
anns = self.coco.loadAnns(ids=ann_ids)
# parse annotations into a list
cat_ids = []
bbox_values = []
segmentation = []
for ann in anns:
if ann['iscrowd'] != 1:
cat_ids.append(ann['category_id'])
bbox_values.append(ann['bbox'])
segmentation.append(ann['segmentation'])
if not (coco_labels):
cats = self.coco.loadCats(ids=cat_ids)
cat_ids = [self.BATCH_CLASSES.index(cat['name']) for cat in cats]
return cat_ids, bbox_values, segmentation
def get_sample(self):
while True:
# 1. Get img_id
img_id = self.__provide_img_id(self.samples_read_counter)
# 2. Get annotatinos
labels, gtbboxes, segmentation = self.__provide_img_tags(img_id)
self.samples_read_counter += 1
# 2. Read the file name
file_name = self.__provide_img_file_name(img_id)
file_path = os.path.join(self.IMAGES_PATH, file_name)
try:
im = self.resize.imResize(self.imread.read(file_path))
w, h, c = im.shape
im = np.reshape(im, [-1, c])
mean = np.mean(im, axis=0, dtype=np.float32)
im = np.subtract(im, mean)
std = np.std(im, axis=0, dtype=np.float32)
im = np.divide(im, std)
im = np.reshape(im, [w, h, c])
gtbboxes = [
self.resize.bboxResize(gtbbox) for gtbbox in gtbboxes
]
# provide anchor ids for each image
aids = self.find_anchor_ids(gtbboxes)
# calculate deltas for each anchor and add them to the delta_per_batch
deltas = self.estimate_deltas(gtbboxes, aids)
break
except:
pass
dense_anns = self.__map_to_grid(labels, gtbboxes, aids, deltas)
"""
im = tf.convert_to_tensor(im, dtype=tf.float32, name='image')
labels = tf.convert_to_tensor(dense_anns['dense_labels'], dtype=tf.int32, name='labels')
aids = tf.convert_to_tensor(dense_anns['masks'], dtype=tf.int32, name='aids')
deltas = tf.convert_to_tensor(dense_anns['bbox_deltas'], dtype=tf.float32, name='bbox_deltas')
bbox_values = tf.convert_to_tensor(dense_anns['bbox_values'], dtype=tf.float32, name='bbox_values')
"""
im = im
labels = dense_anns['dense_labels']
aids = dense_anns['masks']
deltas = dense_anns['bbox_deltas']
bbox_values = dense_anns['bbox_values']
return [im, labels, aids, deltas, bbox_values]
def __map_to_grid(self, sparse_label, sparse_gtbox, sparse_aids,
sparse_deltas):
# Convert into a flattened out list
label_indices, \
bbox_indices, \
box_delta_values, \
mask_indices, \
box_values = convertToFixedSize(aidx=sparse_aids,
labels=sparse_label,
boxes_deltas=sparse_deltas,
bboxes=sparse_gtbox)
# Extract variables to make it more readable
n_anchors = len(self.ANCHOR_BOX)
n_classes = len(self.CLASS_NAMES_AVAILABLE) #len(self.BATCH_CLASSES)
n_labels = len(label_indices)
# Dense boxes
label_indices = sparse_to_dense(label_indices, [n_anchors, n_classes],
np.ones(n_labels, dtype=np.float))
bbox_deltas = sparse_to_dense(bbox_indices, [n_anchors, 4],
box_delta_values)
mask = np.reshape(
sparse_to_dense(mask_indices, [n_anchors],
np.ones(n_labels, dtype=np.float)), [n_anchors, 1])
box_values = sparse_to_dense(bbox_indices, [n_anchors, 4], box_values)
return {
'dense_labels': label_indices,
'masks': mask,
'bbox_deltas': bbox_deltas,
'bbox_values': box_values
}
from PythonAPI.pycocotools.coco import COCO
import numpy as np
dataDir = '.'
dataType = 'train2014'
annFile = '{}/annotations/captions_{}.json'.format(dataDir, dataType)
coco_caps = COCO(annFile)
def getCaption(img_name):
print("Get img:", img_name)
IMG_ID = int(img_name.split("_")[-1].split(".")[0])
imgIds = coco_caps.getImgIds(imgIds=[IMG_ID])
img = coco_caps.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0]
annIds = coco_caps.getAnnIds(imgIds=img['id'])
anns = coco_caps.loadAnns(annIds)
captions_list = coco_caps.showAnns(anns)
str_caption = ""
i = 1
for oneSen in captions_list:
str_caption += str(i)
str_caption += ". "
str_caption += oneSen
str_caption += '\n'
i += 1
return str_caption[:-1]
if __name__ == "__main__":
ret = getCaption("COCO_train2014_000000000009.jpg")
import sys
sys.path.append("PythonAPI/")
from PythonAPI.pycocotools.coco import COCO
import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
dataDir = '/media/reynaldo/Data/Databases/coco/coco_2017'
dataType = {'train': 'train2017', 'val': 'val2017'}
annThingsFile = '%s/annotations/instances_%s.json' % (dataDir, dataType["val"])
annStuffFile = '%s/annotations/stuff_%s.json' % (dataDir, dataType["val"])
# initialize COCO api for instance annotations
coco_things = COCO(annThingsFile)
coco_stuff = COCO(annStuffFile)
# display COCO categories and supercategories
thing_cats = coco_things.loadCats(coco_things.getCatIds())
stuff_cats = coco_stuff.loadCats(coco_stuff.getCatIds())
# Things Classes #
nms = [cat['name'] for cat in thing_cats]
print('\nCOCO Things Categories: \n\n', ' '.join(nms))
nms = set([cat['supercategory'] for cat in thing_cats])
print('\nCOCO Things Supercategories: \n', ' '.join(nms))
things_count = 0