def load_coco(self, dataset_dir, subset, class_ids=None, class_map=None, return_coco=False): """Load a subset of the COCO dataset. dataset_dir: The root directory of the COCO dataset. subset: What to load (train, val, minival, val35k) class_ids: If provided, only loads images that have the given classes. class_map: TODO: Not implemented yet. Supports maping classes from different datasets to the same class ID. return_coco: If True, returns the COCO object. """ # Path image_dir = os.path.join( dataset_dir, "train2017" if subset == "train" else "val2017") # Create COCO object json_path_dict = { "train": "annotations/instances_train2017.json", "val": "annotations/instances_val2017.json" # "minival": "annotations/instances_minival2017.json", # "val35k": "annotations/instances_valminusminival2017.json", } coco = COCO(os.path.join(dataset_dir, json_path_dict[subset])) # Load all classes or a subset? if not class_ids: # All classes class_ids = sorted(coco.getCatIds()) # All images or a subset? if class_ids: image_ids = [] for id in class_ids: image_ids.extend(list(coco.getImgIds(catIds=[id]))) # Remove duplicates image_ids = list(set(image_ids)) else: # All images image_ids = list(coco.imgs.keys()) # Add classes for i in class_ids: self.add_class("coco", i, coco.loadCats(i)[0]["name"]) # Add images for i in image_ids: self.add_image("coco", image_id=i, path=os.path.join(image_dir, coco.imgs[i]['file_name']), width=coco.imgs[i]["width"], height=coco.imgs[i]["height"], annotations=coco.loadAnns( coco.getAnnIds(imgIds=[i], iscrowd=False))) if return_coco: return coco
def __init__(self, coco_name, main_controller, shuffle=True, resize_dim=(1024, 1024)): """ COCO class is an adapter for coco dataset that ensures campatibility with ConvDet layer logic. The dataset should be initialized with: :param name: string with a name of the dataset. Good names can be 'train', 'test', or 'va', but I am not a stickler =) :param path: a full path to the folder containing images and annotations folder :param mc: main controller containing remaining parameters necessary for the proper initialization. mc should contain: - mc.batch_size - an integer greater than 0 - mc.ANNOTATIONS_FILE_NAME - a file name located in the coco_path/annotations - mc.BATCH_CLASSES - an array of classes to be learned (at least 1) """ self.shuffle = shuffle self.imgIds = [] self.catIds = [] self.name = coco_name IMDB.__init__(self, resize_dim=resize_dim, feature_map_size=main_controller.OUTPUT_RES, main_controller=main_controller) #1. Get an array of image indicies assert type(main_controller.ANNOTATIONS_FILE_NAME)==str,\ "Provide a name of the file containing annotations in mc.ANNOTATIONS_FILE_NAME" self.annotations_file = main_controller.ANNOTATIONS_FILE_NAME self.coco = COCO(self.annotations_file) categories = self.coco.loadCats(self.coco.getCatIds()) self.CLASS_NAMES_AVAILABLE = [ category['name'] for category in categories ] self.CATEGORIES = set( [category['supercategory'] for category in categories]) assert type(main_controller.BATCH_CLASSES) == list and len(main_controller.BATCH_CLASSES)>0,\ "Provide a list of classes to be learned in this batch through mc.BATCH_CLASSES" self.BATCH_CLASSES = main_controller.BATCH_CLASSES
def main(train_type='Resnet'): val_annFile = './annotations/instances_val2017.json' val_coco = COCO(val_annFile) val_set, val_len = prepare_dataset(val_coco, batch_size, [224, 224]) handle = tf.placeholder(tf.string, []) iterator = tf.data.Iterator.from_string_handle( handle, (tf.float32, tf.float32, tf.float32), ((None, None, None, 3), (None, num_classes), (None, None, None, num_classes)) ) imgs, labels, gt = iterator.get_next() val_iter = val_set.make_initializable_iterator() val_handle = val_iter.string_handle() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) val_handle = sess.run(val_handle) print('initial done') tic = time.time() if train_type == 'Resnet': cnt = 0 total_loss = 0.0 sess.run(val_iter.initializer) while True: try: img, y = sess.run([imgs, labels], feed_dict={handle: val_handle}) cnt += 16 print('total', cnt, img.shape, y.shape) if cnt >= 128: break except tf.errors.OutOfRangeError: break else: pass print('Done (t={:0.2f}s)'.format(time.time()- tic))
import sys, os # sys.path.insert(0, os.path.abspath('.')) sys.path.insert(0, './PythonAPI/') # sys.path.insert(0, os.path.abspath('data')) for _ in sys.path: print(_) from PythonAPI.pycocotools.coco import COCO import cv2 import numpy as np import os from libs.label_name_dict import coco_dict annotation_path = '/home/yjr/DataSet/COCO/2017/annotations/instances_train2017.json' print("load coco .... it will cost about 17s..") coco = COCO(annotation_path) imgId_list = coco.getImgIds() imgId_list = np.array(imgId_list) total_imgs = len(imgId_list) # print (NAME_LABEL_DICT) def next_img(step): if step % total_imgs == 0: np.random.shuffle(imgId_list) imgid = imgId_list[step % total_imgs]
def main(train_type='Resnet', restore=False, maxiter=10, test=False): train_annFile = './annotations/instances_train2017.json' val_annFile = './annotations/instances_val2017.json' with open('PythonAPI/cat_dict.json', 'r') as f: cat_dict = json.load(f) train_coco = COCO(train_annFile) val_coco = COCO(val_annFile) train_key = [] val_key = [] for i in list(range(num_classes - 1)): train_key.append(train_coco.cats[cat_dict['c2id'][str(i)]]['name']) val_key.append(val_coco.cats[cat_dict['c2id'][str(i)]]['name']) train_key.append('background') val_key.append('background') train_set, train_len = prepare_dataset(train_coco, batch_size, [224, 224]) val_set, val_len = prepare_dataset(val_coco, batch_size, [224, 224]) handle = tf.placeholder(tf.string, []) iterator = tf.data.Iterator.from_string_handle( handle, (tf.float32, tf.float32, tf.float32), ((None, None, None, 3), (None, num_classes), (None, None, None, num_classes))) imgs, labels, gt = iterator.get_next() train_iter = train_set.make_initializable_iterator() val_iter = val_set.make_initializable_iterator() train_handle = train_iter.string_handle() val_handle = val_iter.string_handle() resnet_step = tf.Variable(0, dtype=tf.int64, name="resnet_step", trainable=False) deep_step = tf.Variable(0, dtype=tf.int64, name="deep_step", trainable=False) crf_step = tf.Variable(0, dtype=tf.int64, name="crf_step", trainable=False) crf_seperate = tf.placeholder(tf.bool, shape=[]) _deeplab = deeplab.deeplab_v3_plus(imgs, [128, 64], [48, num_classes], num_classes) res_out = _deeplab.get_dense() res_loss = tf.nn.softmax_cross_entropy_with_logits_v2( labels=tf.stop_gradient(labels), logits=res_out, name='res_loss') res_mean_loss = tf.reduce_mean(res_loss) res_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize( res_loss, global_step=resnet_step) pred_out = _deeplab.get_pred() pred_softmax = tf.nn.softmax(pred_out) pred_loss = tf.nn.softmax_cross_entropy_with_logits_v2( labels=tf.stop_gradient(gt), logits=pred_out, name='pred_loss') pred_mean_loss = tf.reduce_mean(pred_loss) pred_op = tf.train.AdamOptimizer(learning_rate=1e-4).minimize( pred_loss, global_step=deep_step) pred_acc = tf.reduce_mean( tf.cast(tf.argmax(pred_out, -1) == tf.argmax(gt, -1), tf.float32)) crf_in = tf.cond(crf_seperate, lambda: tf.stop_gradient(pred_out), lambda: pred_out) crf_kernel = tf.constant([1., 1., 1., .5, .5], dtype=tf.float32) crf_out = crf_rnn(crf_in, imgs, crf_kernel, 3, "crf_rnn") crf_loss = tf.nn.softmax_cross_entropy_with_logits_v2( labels=tf.stop_gradient(gt), logits=crf_out, name='crf_loss') crf_mean_loss = tf.reduce_mean(crf_loss) crf_op = tf.train.AdamOptimizer(learning_rate=2e-5).minimize( crf_loss, global_step=crf_step) crf_acc = tf.reduce_mean( tf.cast(tf.argmax(crf_out, -1) == tf.argmax(gt, -1), tf.float32)) crf_softmax = tf.nn.softmax(crf_out) reader = pywrap_tensorflow.NewCheckpointReader(model_path) restore_dict = dict() for v in tf.get_collection(tf.GraphKeys.VARIABLES): tname = v.name.split(':')[0] if reader.has_tensor(tname): restore_dict[tname] = v saver = tf.train.Saver() restorer = tf.train.Saver(restore_dict) summary = summarizer(os.path.join( log_dir, 'log%s.csv' % ('CRF' if train_type == 'CRF-only' else train_type)), ['step', 'train_loss', 'val_loss'], 25, restore=restore) config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: if restore: sess.run(tf.global_variables_initializer()) restorer.restore(sess, model_path) print('restored') else: sess.run(tf.global_variables_initializer()) print('initial done') train_handle = sess.run(train_handle) val_handle = sess.run(val_handle) if test: if train_type == "Deep": pair = [imgs, pred_softmax] elif train_type in ["CRF-only", "CRF"]: pair = [imgs, crf_softmax] else: pair = [imgs, pred_softmax] sess.run(val_iter.initializer) cnt = 0 for epoc in range(100): img, pred = sess.run(pair, feed_dict={handle: val_handle}) print('iter%d' % epoc) for i in range(img.shape[0]): plot.draw_raw_image(img[i][:, :, ::-1], "./test/img_%d_raw.jpg" % cnt) plot.draw_image(pred[i], "./test/img_%d_pred.jpg" % cnt) cnt += 1 return sess.run([train_iter.initializer, val_iter.initializer]) if train_type == 'Resnet': cnt = 0 save_cnt = 0 epoc = 0 while epoc < maxiter: _, _loss = sess.run([res_op, res_mean_loss], feed_dict={handle: train_handle}) cnt += batch_size save_cnt += 1 if summary.step == summary.steps - 1: print("%d/%d %f" % (cnt, train_len, cnt / train_len)) _valloss = sess.run(res_mean_loss, feed_dict={handle: val_handle}) summary.summary(train_loss=_loss, val_loss=_valloss, step=sess.run(resnet_step)) else: summary.summary(train_loss=_loss) if cnt >= train_len: epoc += 1 print('epoc %d done' % epoc) cnt -= train_len saver.save(sess, model_path) print('model saved') save_cnt = 0 elif save_cnt >= 50: save_cnt = 0 saver.save(sess, model_path) print('model saved') else: pass elif train_type == 'Deep': cnt = 0 save_cnt = 0 epoc = 0 while epoc < maxiter: _, _loss = sess.run([pred_op, pred_mean_loss], feed_dict={handle: train_handle}) cnt += batch_size save_cnt += 1 if summary.step == summary.steps - 1: print("%d/%d %f" % (cnt, train_len, cnt / train_len)) _valloss = sess.run(pred_mean_loss, feed_dict={handle: val_handle}) summary.summary(train_loss=_loss, val_loss=_valloss, step=sess.run(deep_step)) else: summary.summary(train_loss=_loss) if cnt >= train_len: epoc += 1 print('epoc %d done' % epoc) cnt -= train_len saver.save(sess, model_path) print('model saved') save_cnt = 0 elif save_cnt >= 50: save_cnt = 0 saver.save(sess, model_path) print('model saved') else: pass elif train_type in ["CRF-only", "CRF"]: crf_only = (train_type == "CRF-only") cnt = 0 save_cnt = 0 epoc = 0 while epoc < maxiter: _, _loss = sess.run([crf_op, crf_mean_loss], feed_dict={ handle: train_handle, crf_seperate: crf_only }) cnt += batch_size save_cnt += 1 if summary.step == summary.steps - 1: print("%d/%d %f" % (cnt, train_len, cnt / train_len)) _valloss = sess.run(crf_mean_loss, feed_dict={ handle: val_handle, crf_seperate: crf_only }) summary.summary(train_loss=_loss, val_loss=_valloss, step=sess.run(crf_step)) else: summary.summary(train_loss=_loss) if cnt >= train_len: epoc += 1 print('epoc %d done' % epoc) cnt -= train_len saver.save(sess, model_path) print('model saved') save_cnt = 0 elif save_cnt >= 50: save_cnt = 0 saver.save(sess, model_path) print('model saved') else: pass else: assert False, "unknown training type %s" % train_type
from PythonAPI.pycocotools.coco import COCO import numpy as np import skimage.io as io import matplotlib.pyplot as plt import pylab pylab.rcParams['figure.figsize'] = (8.0, 10.0) dataDir='/home/home/AndroidStudioProjects/Messaging/visible/coco_dataset_images' dataType='val2017' annFile='{}/annotations/instances_{}.json'.format(dataDir,dataType) # initialize COCO api for instance annotations coco=COCO(annFile)
class coco(IMDB): def __init__(self, coco_name, main_controller, shuffle=True, resize_dim=(1024, 1024)): """ COCO class is an adapter for coco dataset that ensures campatibility with ConvDet layer logic. The dataset should be initialized with: :param name: string with a name of the dataset. Good names can be 'train', 'test', or 'va', but I am not a stickler =) :param path: a full path to the folder containing images and annotations folder :param mc: main controller containing remaining parameters necessary for the proper initialization. mc should contain: - mc.batch_size - an integer greater than 0 - mc.ANNOTATIONS_FILE_NAME - a file name located in the coco_path/annotations - mc.BATCH_CLASSES - an array of classes to be learned (at least 1) """ self.shuffle = shuffle self.imgIds = [] self.catIds = [] self.name = coco_name IMDB.__init__(self, resize_dim=resize_dim, feature_map_size=main_controller.OUTPUT_RES, main_controller=main_controller) #1. Get an array of image indicies assert type(main_controller.ANNOTATIONS_FILE_NAME)==str,\ "Provide a name of the file containing annotations in mc.ANNOTATIONS_FILE_NAME" self.annotations_file = main_controller.ANNOTATIONS_FILE_NAME self.coco = COCO(self.annotations_file) categories = self.coco.loadCats(self.coco.getCatIds()) self.CLASS_NAMES_AVAILABLE = [ category['name'] for category in categories ] self.CATEGORIES = set( [category['supercategory'] for category in categories]) assert type(main_controller.BATCH_CLASSES) == list and len(main_controller.BATCH_CLASSES)>0,\ "Provide a list of classes to be learned in this batch through mc.BATCH_CLASSES" self.BATCH_CLASSES = main_controller.BATCH_CLASSES @property def imgIds(self): return self.__imgIds @imgIds.setter def imgIds(self, values): assert type(values)==list,\ "imgIds is incorrect. Array is expected. {} was recieved.".format(type(values).__name__) shuffle_flag = self.shuffle if shuffle_flag: shuffle(values) self.__imgIds = values @property def annotations_file(self): assert os.path.exists(self.__annotations_file),\ "Invalid path was provided for the data set. The following path doesn't exist: {}"\ .format(self.__annotations_file) return self.__annotations_file @annotations_file.setter def annotations_file(self, value): assert os.path.exists(value), "Annotations file doesn't exis at path {}. Please provide a full path to annotatinos." \ .format(value) self.__annotations_file = value BATCH_CLASSES = None @property def BATCH_CLASSES(self): return self.__BATCH_CLASSES @BATCH_CLASSES.setter def BATCH_CLASSES(self, values): for value in values: assert value in self.CLASS_NAMES_AVAILABLE,\ "BATCH_CLASSES array is incorrect. The following class (from the array) " + \ "is not present in self.CLASS_NAMES_AVAILABLE array: {}.".\ format(value) self.__BATCH_CLASSES = values # build an array of file indicies img_ids = [] cat_ids = [] for class_name in self.__BATCH_CLASSES: cat_id = self.coco.getCatIds(catNms=[class_name]) cat_ids.extend(cat_id) img_ids.extend(self.coco.getImgIds(catIds=cat_id)) # update image ids self.imgIds = img_ids self.catIds = cat_ids def __provide_img_id(self, id): return self.imgIds[id] def epoch_size(self): return len(self.imgIds) def __provide_img_file_name(self, id): """ Protocol describing the implementation of a method that provides the name of the image file based on an image id. :param id: dataset specific image id :return: string containing file name """ descriptions = self.coco.loadImgs(id)[0] return descriptions['file_name'] def __provide_img_tags(self, id, coco_labels=True): """ Protocol describing the implementation of a method that provides tags for the image file based on an image id. :param id: dataset specific image id :param coco_labels(optional): indicates wherher coco label ids should be returned default value is False - BATCH_CLASS ids should be returned :return: an array containing the list of tags """ # Extract annotation ids ann_ids = self.coco.getAnnIds(imgIds=[id], catIds=self.catIds, iscrowd=None) # get all annotations available anns = self.coco.loadAnns(ids=ann_ids) # parse annotations into a list cat_ids = [] bbox_values = [] segmentation = [] for ann in anns: if ann['iscrowd'] != 1: cat_ids.append(ann['category_id']) bbox_values.append(ann['bbox']) segmentation.append(ann['segmentation']) if not (coco_labels): cats = self.coco.loadCats(ids=cat_ids) cat_ids = [self.BATCH_CLASSES.index(cat['name']) for cat in cats] return cat_ids, bbox_values, segmentation def get_sample(self): while True: # 1. Get img_id img_id = self.__provide_img_id(self.samples_read_counter) # 2. Get annotatinos labels, gtbboxes, segmentation = self.__provide_img_tags(img_id) self.samples_read_counter += 1 # 2. Read the file name file_name = self.__provide_img_file_name(img_id) file_path = os.path.join(self.IMAGES_PATH, file_name) try: im = self.resize.imResize(self.imread.read(file_path)) w, h, c = im.shape im = np.reshape(im, [-1, c]) mean = np.mean(im, axis=0, dtype=np.float32) im = np.subtract(im, mean) std = np.std(im, axis=0, dtype=np.float32) im = np.divide(im, std) im = np.reshape(im, [w, h, c]) gtbboxes = [ self.resize.bboxResize(gtbbox) for gtbbox in gtbboxes ] # provide anchor ids for each image aids = self.find_anchor_ids(gtbboxes) # calculate deltas for each anchor and add them to the delta_per_batch deltas = self.estimate_deltas(gtbboxes, aids) break except: pass dense_anns = self.__map_to_grid(labels, gtbboxes, aids, deltas) """ im = tf.convert_to_tensor(im, dtype=tf.float32, name='image') labels = tf.convert_to_tensor(dense_anns['dense_labels'], dtype=tf.int32, name='labels') aids = tf.convert_to_tensor(dense_anns['masks'], dtype=tf.int32, name='aids') deltas = tf.convert_to_tensor(dense_anns['bbox_deltas'], dtype=tf.float32, name='bbox_deltas') bbox_values = tf.convert_to_tensor(dense_anns['bbox_values'], dtype=tf.float32, name='bbox_values') """ im = im labels = dense_anns['dense_labels'] aids = dense_anns['masks'] deltas = dense_anns['bbox_deltas'] bbox_values = dense_anns['bbox_values'] return [im, labels, aids, deltas, bbox_values] def __map_to_grid(self, sparse_label, sparse_gtbox, sparse_aids, sparse_deltas): # Convert into a flattened out list label_indices, \ bbox_indices, \ box_delta_values, \ mask_indices, \ box_values = convertToFixedSize(aidx=sparse_aids, labels=sparse_label, boxes_deltas=sparse_deltas, bboxes=sparse_gtbox) # Extract variables to make it more readable n_anchors = len(self.ANCHOR_BOX) n_classes = len(self.CLASS_NAMES_AVAILABLE) #len(self.BATCH_CLASSES) n_labels = len(label_indices) # Dense boxes label_indices = sparse_to_dense(label_indices, [n_anchors, n_classes], np.ones(n_labels, dtype=np.float)) bbox_deltas = sparse_to_dense(bbox_indices, [n_anchors, 4], box_delta_values) mask = np.reshape( sparse_to_dense(mask_indices, [n_anchors], np.ones(n_labels, dtype=np.float)), [n_anchors, 1]) box_values = sparse_to_dense(bbox_indices, [n_anchors, 4], box_values) return { 'dense_labels': label_indices, 'masks': mask, 'bbox_deltas': bbox_deltas, 'bbox_values': box_values }
from PythonAPI.pycocotools.coco import COCO import numpy as np dataDir = '.' dataType = 'train2014' annFile = '{}/annotations/captions_{}.json'.format(dataDir, dataType) coco_caps = COCO(annFile) def getCaption(img_name): print("Get img:", img_name) IMG_ID = int(img_name.split("_")[-1].split(".")[0]) imgIds = coco_caps.getImgIds(imgIds=[IMG_ID]) img = coco_caps.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0] annIds = coco_caps.getAnnIds(imgIds=img['id']) anns = coco_caps.loadAnns(annIds) captions_list = coco_caps.showAnns(anns) str_caption = "" i = 1 for oneSen in captions_list: str_caption += str(i) str_caption += ". " str_caption += oneSen str_caption += '\n' i += 1 return str_caption[:-1] if __name__ == "__main__": ret = getCaption("COCO_train2014_000000000009.jpg")
import sys sys.path.append("PythonAPI/") from PythonAPI.pycocotools.coco import COCO import numpy as np import skimage.io as io import matplotlib.pyplot as plt import pylab dataDir = '/media/reynaldo/Data/Databases/coco/coco_2017' dataType = {'train': 'train2017', 'val': 'val2017'} annThingsFile = '%s/annotations/instances_%s.json' % (dataDir, dataType["val"]) annStuffFile = '%s/annotations/stuff_%s.json' % (dataDir, dataType["val"]) # initialize COCO api for instance annotations coco_things = COCO(annThingsFile) coco_stuff = COCO(annStuffFile) # display COCO categories and supercategories thing_cats = coco_things.loadCats(coco_things.getCatIds()) stuff_cats = coco_stuff.loadCats(coco_stuff.getCatIds()) # Things Classes # nms = [cat['name'] for cat in thing_cats] print('\nCOCO Things Categories: \n\n', ' '.join(nms)) nms = set([cat['supercategory'] for cat in thing_cats]) print('\nCOCO Things Supercategories: \n', ' '.join(nms)) things_count = 0