from datasets.nist import nist from datasets.dvmm import dvmm from datasets.swapme import swapme import numpy as np # Set up voc_<year>_<split> for year in ['2007', '2012']: for split in ['train', 'val', 'trainval', 'test']: name = 'voc_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: pascal_voc(split, year)) # Set up coco_2014_<split> for year in ['2014']: for split in ['train', 'val', 'minival', 'valminusminival', 'trainval']: name = 'coco_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) # Set up coco_2015_<split> for year in ['2015']: for split in ['test', 'test-dev']: name = 'COCO_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) dvmm_path = '/vulcan/scratch/pengzhou/dataset/4cam_splc' for split in ['dist_train', 'dist_test']: name = split __sets[name] = (lambda split=split: dvmm(split, 2007, dvmm_path)) dso_path = '/vulcan/scratch/pengzhou/dataset/COVERAGE' for split in ['dist_cover_train_single', 'dist_cover_test_single']: name = split
from datasets.pascal_voc import pascal_voc from datasets.coco import coco from datasets.grotoap2 import grotoap2 import numpy as np # Set up voc_<year>_<split> using selective search "fast" mode for year in ['2007', '2012']: for split in ['train', 'val', 'trainval', 'test']: name = 'voc_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: pascal_voc(split, year)) # Set up coco_2014_<split> for year in ['2014']: for split in ['train', 'val', 'minival', 'valminusminival']: name = 'coco_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) # Set up coco_2015_<split> for year in ['2015']: for split in ['test', 'test-dev']: name = 'coco_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) # Set up grotoap2 for split in ['train', 'val', 'trainval', 'test']: name = 'grotoap2_{}'.format(split) __sets[name] = (lambda split=split: grotoap2(split)) def get_imdb(name): """Get an imdb (image database) by name."""
def test_scene_model(config): output_dir = osp.join(config.model_dir, 'test_scene_model') maybe_create(output_dir) plt.switch_backend('agg') transformer = image_normalize('background') db = coco(config, 'train', transform=transformer) pca_table = AllCategoriesTables(db) pca_table.run_PCAs_and_build_nntables_in_feature_space() loader = DataLoader(db, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) net = SceneModel(db) net.eval() for cnt, batched in enumerate(loader): word_inds = batched['word_inds'].long() word_lens = batched['word_lens'].long() bg_images = batched['background'].float() fg_inds = batched['fg_inds'].long() gt_inds = batched['out_inds'].long() gt_vecs = batched['out_vecs'].float() gt_msks = batched['out_msks'].float() fg_onehots = indices2onehots(fg_inds, config.output_vocab_size) # inf_outs, _ = net((word_inds, word_lens, bg_images, fg_onehots)) # obj_logits, coord_logits, attri_logits, pca_vectors, enc_msks, what_wei, where_wei = inf_outs # print('teacher forcing') # print('obj_logits ', obj_logits.size()) # print('coord_logits ', coord_logits.size()) # print('attri_logits ', attri_logits.size()) # print('pca_vectors ', pca_vectors.size()) # if config.what_attn: # print('what_att_logits ', what_wei.size()) # if config.where_attn > 0: # print('where_att_logits ', where_wei.size()) # print('----------------------') inf_outs, env = net.inference(word_inds, word_lens, -1, 0, 0, gt_inds, gt_vecs) # inf_outs, env = net.inference(word_inds, word_lens, -1, 2.0, 0, None, None) obj_logits, coord_logits, attri_logits, pca_vectors, enc_msks, what_wei, where_wei = inf_outs print('scheduled sampling') print('obj_logits ', obj_logits.size()) print('coord_logits ', coord_logits.size()) print('attri_logits ', attri_logits.size()) print('pca_vectors ', pca_vectors.size()) if config.what_attn: print('what_att_logits ', what_wei.size()) if config.where_attn > 0: print('where_att_logits ', where_wei.size()) print('----------------------') sequences = env.batch_redraw(True) for i in range(len(sequences)): sequence = sequences[i] image_idx = batched['image_index'][i] name = '%03d_'%i + str(image_idx).zfill(12) out_path = osp.join(output_dir, name+'.png') color = cv2.imread(batched['color_path'][i], cv2.IMREAD_COLOR) color, _, _ = create_squared_image(color) fig = plt.figure(figsize=(32, 16)) plt.suptitle(batched['sentence'][i], fontsize=30) for j in range(min(len(sequence), 14)): plt.subplot(3, 5, j+1) partially_completed_img = clamp_array(sequence[j], 0, 255).astype(np.uint8) partially_completed_img = partially_completed_img[:,:,::-1] plt.imshow(partially_completed_img) plt.axis('off') plt.subplot(3, 5, 15) plt.imshow(color[:,:,::-1]) plt.axis('off') fig.savefig(out_path, bbox_inches='tight') plt.close(fig) break
from datasets.pascal_voc import pascal_voc from datasets.coco import coco from datasets.detect_task import detect_task import numpy as np # Set up voc_<year>_<split> using selective search "fast" mode for year in ['2007', '2012']: for split in ['train', 'val', 'trainval', 'test']: name = 'voc_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: pascal_voc(split, year)) # Set up coco_2014_<split> for year in ['2014']: for split in ['train', 'val', 'minival', 'valminusminival']: name = 'coco_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) # Set up coco_2015_<split> for year in ['2015']: for split in ['test', 'test-dev']: name = 'coco_{}_{}'.format(year, split) __sets[name] = (lambda split=split, year=year: coco(split, year)) def get_imdb(name): """Get an imdb (image database) by name.""" if not __sets.has_key(name): __sets[name] = (lambda task_name=name: detect_task(task_name)) return __sets[name]() # raise KeyError('Unknown dataset: {}'.format(name)) return __sets[name]()
def test_step_by_step(config): db = coco(config, 'train', '2017') output_dir = osp.join(config.model_dir, 'test_step_by_step') maybe_create(output_dir) all_tables = AllCategoriesTables(db) all_tables.build_nntables_for_all_categories(True) seq_db = sequence_loader(db, all_tables) env = simulator(db, config.batch_size, all_tables) env.reset() loader = DataLoader(seq_db, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) for cnt, batched in enumerate(loader): out_inds = batched['out_inds'].long().numpy() out_vecs = batched['out_vecs'].float().numpy() sequences = [] for i in range(out_inds.shape[1]): frames = env.batch_render_to_pytorch(out_inds[:, i], out_vecs[:, i]) sequences.append(frames) sequences = torch.stack(sequences, dim=1) # sequences = [tensors_to_vols(x) for x in sequences] for i in range(len(sequences)): sequence = sequences[i] image_idx = batched['image_index'][i] name = '%03d_' % i + str(image_idx).zfill(12) out_path = osp.join(output_dir, name + '.png') color = cv2.imread(batched['image_path'][i], cv2.IMREAD_COLOR) color, _, _ = create_squared_image(color) fig = plt.figure(figsize=(32, 32)) plt.suptitle(batched['sentence'][i], fontsize=30) for j in range(min(len(sequence), 14)): plt.subplot(4, 4, j + 1) seq_np = sequence[j].cpu().data.numpy() if config.use_color_volume: partially_completed_img, _ = heuristic_collage(seq_np, 83) else: partially_completed_img = seq_np[:, :, -3:] partially_completed_img = clamp_array(partially_completed_img, 0, 255).astype(np.uint8) partially_completed_img = partially_completed_img[:, :, ::-1] plt.imshow(partially_completed_img) plt.axis('off') plt.subplot(4, 4, 16) plt.imshow(color[:, :, ::-1]) plt.axis('off') fig.savefig(out_path, bbox_inches='tight') plt.close(fig) break
def test_coco_dataloader(config): db = coco(config, 'train', '2017') all_tables = AllCategoriesTables(db) all_tables.build_nntables_for_all_categories(True) sequence_db = sequence_loader(db, all_tables) output_dir = osp.join(config.model_dir, 'test_coco_dataloader') maybe_create(output_dir) loader = DataLoader(sequence_db, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers) start = time() for cnt, batched in enumerate(loader): x = batched['background'].float() y = batched['foreground'].float() z = batched['negative'].float() # x = sequence_onehot_volumn_preprocess(x, len(db.classes)) x = sequence_color_volumn_preprocess(x, len(db.classes)) y = sequence_onehot_volumn_preprocess(y, len(db.classes)) z = sequence_onehot_volumn_preprocess(z, len(db.classes)) # cv2.imwrite('mask0.png', y[0,2,:,:,-4].cpu().data.numpy()) # cv2.imwrite('mask1.png', y[1,2,:,:,-4].cpu().data.numpy()) # cv2.imwrite('mask2.png', y[2,2,:,:,-4].cpu().data.numpy()) # cv2.imwrite('mask3.png', y[3,2,:,:,-4].cpu().data.numpy()) # cv2.imwrite('label0.png', y[0,2,:,:,3].cpu().data.numpy()) # cv2.imwrite('label1.png', y[1,2,:,:,3].cpu().data.numpy()) # cv2.imwrite('label2.png', y[2,2,:,:,3].cpu().data.numpy()) # cv2.imwrite('label3.png', y[3,2,:,:,3].cpu().data.numpy()) # cv2.imwrite('color0.png', y[0,2,:,:,-3:].cpu().data.numpy()) # cv2.imwrite('color1.png', y[1,2,:,:,-3:].cpu().data.numpy()) # cv2.imwrite('color2.png', y[2,2,:,:,-3:].cpu().data.numpy()) # cv2.imwrite('color3.png', y[3,2,:,:,-3:].cpu().data.numpy()) # cv2.imwrite('bg0.png', x[0,3,:,:,9:12].cpu().data.numpy()) # cv2.imwrite('bg1.png', x[1,3,:,:,9:12].cpu().data.numpy()) # cv2.imwrite('bg2.png', x[2,3,:,:,9:12].cpu().data.numpy()) # cv2.imwrite('bg3.png', x[3,3,:,:,9:12].cpu().data.numpy()) x = (x - 128.0).permute(0, 1, 4, 2, 3) y = (y - 128.0).permute(0, 1, 4, 2, 3) z = (z - 128.0).permute(0, 1, 4, 2, 3) print('background', x.size()) print('foreground', y.size()) print('negative', z.size()) print('word_inds', batched['word_inds'].size()) print('word_lens', batched['word_lens'].size()) print('fg_inds', batched['fg_inds'].size()) print('patch_inds', batched['patch_inds'].size()) print('out_inds', batched['out_inds'].size()) print('out_msks', batched['out_msks'].size()) print('foreground_resnets', batched['foreground_resnets'].size()) print('negative_resnets', batched['negative_resnets'].size()) print('foreground_resnets', batched['foreground_resnets'][0, 0]) print('negative_resnets', batched['negative_resnets'][0, 0]) print('out_msks', batched['out_msks'][0]) print('patch_inds', batched['patch_inds'][0]) plt.switch_backend('agg') bg_images = x fg_images = y neg_images = z bsize, ssize, n, h, w = bg_images.size() bg_images = bg_images.view(bsize * ssize, n, h, w) bg_images = tensors_to_vols(bg_images) bg_images = bg_images.reshape(bsize, ssize, h, w, n) bsize, ssize, n, h, w = fg_images.size() fg_images = fg_images.view(bsize * ssize, n, h, w) fg_images = tensors_to_vols(fg_images) fg_images = fg_images.reshape(bsize, ssize, h, w, n) bsize, ssize, n, h, w = neg_images.size() neg_images = neg_images.view(bsize * ssize, n, h, w) neg_images = tensors_to_vols(neg_images) neg_images = neg_images.reshape(bsize, ssize, h, w, n) for i in range(bsize): bg_seq = bg_images[i] fg_seq = fg_images[i] neg_seq = neg_images[i] image_idx = batched['image_index'][i] fg_inds = batched['fg_inds'][i] name = '%03d_' % i + str(image_idx).zfill(12) out_path = osp.join(output_dir, name + '.png') color = cv2.imread(batched['image_path'][i], cv2.IMREAD_COLOR) color, _, _ = create_squared_image(color) fig = plt.figure(figsize=(48, 32)) plt.suptitle(batched['sentence'][i], fontsize=30) for j in range(min(len(bg_seq), 15)): bg, _ = heuristic_collage(bg_seq[j], 83) bg_mask = 255 * np.ones((bg.shape[1], bg.shape[0])) row, col = np.where(np.sum(np.absolute(bg), -1) == 0) bg_mask[row, col] = 0 # bg = bg_seq[j][:,:,-3:] # bg_mask = bg_seq[j][:,:,-4] bg_mask = np.repeat(bg_mask[..., None], 3, -1) fg_color = fg_seq[j][:, :, -3:] # fg_mask = fg_seq[j][:,:,fg_inds[j+1]] fg_mask = fg_seq[j][:, :, -4] neg_color = neg_seq[j][:, :, -3:] # neg_mask = neg_seq[j][:,:,fg_inds[j+1]] neg_mask = neg_seq[j][:, :, -4] color_pair = np.concatenate((fg_color, neg_color), 1) mask_pair = np.concatenate((fg_mask, neg_mask), 1) mask_pair = np.repeat(mask_pair[..., None], 3, -1) patch = np.concatenate((color_pair, mask_pair), 0) patch = cv2.resize(patch, (bg.shape[1], bg.shape[0])) partially_completed_img = np.concatenate((bg, bg_mask, patch), 1) partially_completed_img = clamp_array(partially_completed_img, 0, 255).astype(np.uint8) partially_completed_img = partially_completed_img[:, :, ::-1] plt.subplot(4, 4, j + 1) plt.imshow(partially_completed_img) plt.axis('off') plt.subplot(4, 4, 16) plt.imshow(color[:, :, ::-1]) plt.axis('off') fig.savefig(out_path, bbox_inches='tight') plt.close(fig) if cnt == 3: break print("Time", time() - start)
def test_puzzle_model(config): output_dir = osp.join(config.model_dir, 'test_puzzle_model') maybe_create(output_dir) plt.switch_backend('agg') db = coco(config, 'train', '2017') all_tables = AllCategoriesTables(db) all_tables.build_nntables_for_all_categories(True) sequence_db = sequence_loader(db, all_tables) loader = DataLoader(sequence_db, batch_size=config.batch_size, shuffle=False, num_workers=config.num_workers) net = PuzzleModel(db) net.eval() for cnt, batched in enumerate(loader): word_inds = batched['word_inds'].long() word_lens = batched['word_lens'].long() bg_images = batched['background'].float() fg_images = batched['foreground'].float() neg_images = batched['negative'].float() fg_resnets = batched['foreground_resnets'].float() neg_resnets = batched['negative_resnets'].float() fg_inds = batched['fg_inds'].long() gt_inds = batched['out_inds'].long() gt_msks = batched['out_msks'].float() fg_onehots = indices2onehots(fg_inds, config.output_vocab_size) inf_outs, _, positive_feats, negative_feats = net((word_inds, word_lens, bg_images, fg_onehots, fg_images, neg_images, fg_resnets, neg_resnets)) obj_logits, coord_logits, attri_logits, patch_vectors, enc_msks, what_wei, where_wei = inf_outs print('teacher forcing') print('obj_logits ', obj_logits.size()) print('coord_logits ', coord_logits.size()) print('attri_logits ', attri_logits.size()) print('patch_vectors ', patch_vectors.size()) print('patch_vectors max:', torch.max(patch_vectors)) print('patch_vectors min:', torch.min(patch_vectors)) print('patch_vectors norm:', torch.norm(patch_vectors, dim=-2)[0,0,0]) print('positive_feats ', positive_feats.size()) print('negative_feats ', negative_feats.size()) if config.what_attn: print('what_att_logits ', what_wei.size()) if config.where_attn > 0: print('where_att_logits ', where_wei.size()) print('----------------------') _, pred_vecs = net.collect_logits_and_vectors(inf_outs, gt_inds) print('pred_vecs', pred_vecs.size()) print('*******************') # # inf_outs, env = net.inference(word_inds, word_lens, -1, 0.0, 0, gt_inds, gt_vecs) # inf_outs, env = net.inference(word_inds, word_lens, -1, 2.0, 0, None, None, all_tables) # obj_logits, coord_logits, attri_logits, patch_vectors, enc_msks, what_wei, where_wei = inf_outs # print('scheduled sampling') # print('obj_logits ', obj_logits.size()) # print('coord_logits ', coord_logits.size()) # print('attri_logits ', attri_logits.size()) # print('patch_vectors ', patch_vectors.size()) # if config.what_attn: # print('what_att_logits ', what_wei.size()) # if config.where_attn > 0: # print('where_att_logits ', where_wei.size()) # print('----------------------') # sequences = env.batch_redraw(True) # for i in range(len(sequences)): # sequence = sequences[i] # image_idx = batched['image_index'][i] # name = '%03d_'%i + str(image_idx).zfill(12) # out_path = osp.join(output_dir, name+'.png') # color = cv2.imread(batched['color_path'][i], cv2.IMREAD_COLOR) # color, _, _ = create_squared_image(color) # fig = plt.figure(figsize=(32, 16)) # plt.suptitle(batched['sentence'][i], fontsize=30) # for j in range(min(len(sequence), 14)): # plt.subplot(3, 5, j+1) # partially_completed_img = clamp_array(sequence[j][:,:,-3:], 0, 255).astype(np.uint8) # partially_completed_img = partially_completed_img[:,:,::-1] # plt.imshow(partially_completed_img) # plt.axis('off') # plt.subplot(3, 5, 15) # plt.imshow(color[:,:,::-1]) # plt.axis('off') # fig.savefig(out_path, bbox_inches='tight') # plt.close(fig) break
from __future__ import absolute_import from __future__ import division from __future__ import print_function import os import os.path as osp import sys this_dir = osp.dirname(__file__) sys.path.insert(0, osp.join(this_dir, '../tools')) import _init_paths from datasets.coco import coco from pprint import pprint import time imdb = coco('minival', '2014') imdb.append_flipped_images() print(len(imdb.roidb)) print(imdb.roidb[0].keys()) # # check image_set and roidb # image_set = loader.image_set # split -> image_ids # roidb = loader.roidb # for split in ['train', 'val', 'testA', 'testB']: # print('[%s] has %s roidb instances and %s images.' % (split, len(roidb), len(image_set[split]))) # # check appending # print('After appending flipped...') # loader.append_flipped_images('train') # for split in ['train', 'val', 'testA', 'testB']: # print('[%s] has %s roidb instances and %s images.' % (split, len(roidb), len(image_set[split])))
name = split __sets[name] = (lambda split=split: dist_fake(split,2007,cover_path)) nist_path='/media/li/Li/NIST2016' for split in ['dist_NIST_train_new_2', 'dist_NIST_test_new_2']: name = split __sets[name] = (lambda split=split: nist(split,2007,nist_path)) casia_path='/media/li/Data/CASIA' #for split in ['casia_train_all_single', 'casia_test_all_1']: for split in ['casia_train_all_single', 'casia_test_all_single']: name = split __sets[name] = (lambda split=split: casia(split,2007,casia_path)) coco_path='/media/li/Li/filter_tamper' for split in ['coco_train_filter_single', 'coco_test_filter_single']: name = split __sets[name] = (lambda split=split: coco(split,2007,coco_path)) def get_imdb(name): """Get an imdb (image database) by name.""" if name not in __sets: raise KeyError('Unknown dataset: {}'.format(name)) return __sets[name]() def list_imdbs(): """List all registered imdbs.""" return list(__sets.keys())
status = subprocess.call(cmd, shell=True) def evaluate_detections(self, all_boxes, output_dir): self._write_voc_results_file(all_boxes) self._do_python_eval(output_dir) #if self.config['matlab_eval']: #self._do_matlab_eval(output_dir) if self.config['cleanup']: for cls in self._classes: if cls == '__background__' or cls == self.classes[0]: continue filename = self._get_voc_results_file_template().format(cls) #os.remove(filename) def competition_mode(self, on): if on: self.config['use_salt'] = False self.config['cleanup'] = False else: self.config['use_salt'] = True self.config['cleanup'] = True if __name__ == '__main__': from datasets.coco import coco d = coco('train', '2007') res = d.roidb from IPython import embed embed()
def __init__(self, image_set, year, devkit_path=None, shuffled=None): imdb.__init__(self, 'merged_' + image_set) self._year = year self._image_set = image_set self._anno_set_dir = image_set if "val" in image_set: self._image_set_dir = "val" if "val1" in image_set: self._anno_set_dir = "val1" if "val2" in image_set: self._anno_set_dir = "val2" elif "train" in image_set: self._anno_set_dir = "train" elif "test" in image_set: self._anno_set_dir = "test" if image_set == "train": self.imdbs = [imagenet(image_set),\ coco(image_set, '2015'),\ cam2(image_set,'2017'),\ sun(image_set,'2012'),\ caltech(image_set,'2009'),\ kitti(image_set,'2013'),\ inria(image_set,'2005'),\ pascal_voc(image_set,'2007'),\ pascal_voc(image_set,'2012')] elif image_set == "test": self.imdbs = [imagenet('val'),\ coco('test-dev', '2015'),\ cam2('all','2017'),\ sun('test','2012'),\ caltech('test','2009'),\ kitti('val','2013'),\ inria('all','2005'),\ pascal_voc('test','2007')] self.roidbs = [None for _ in range(len(self.datasets))] for idx,imdb in enumerate(self.imdbs): self.roidbs[idx] = get_training_roidb(imdb) self._devkit_path = self._get_default_path() if devkit_path is None \ else devkit_path self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year) self._classes = ('__background__', # always index 0 'voc', 'imagenet', 'caltech', 'coco', 'sun', 'kitti', 'inria', 'cam2') self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes))) self._image_ext = '.jpg' self._image_index = self._load_image_set_index() # Default to roidb handler self._roidb_handler = self.selective_search_roidb self._salt = str(uuid.uuid4()) self._comp_id = 'comp4' # PASCAL specific config options self.config = {'cleanup' : True, 'use_salt' : True, 'use_diff' : False, 'matlab_eval' : False, 'rpn_file' : None, 'min_size' : 2} assert os.path.exists(self._devkit_path), \ 'VOCdevkit path does not exist: {}'.format(self._devkit_path) assert os.path.exists(self._data_path), \ 'Path does not exist: {}'.format(self._data_path)